mesh-lab/test/integration/anthropic-bed.test.ts

/**
 * The mesh plays out the model-access refreshable-grant flow for Anthropic end to end, with the
 * vendor's OAuth endpoint STUBBED (novox/hq ADR 0050, Phase C). It proves the one property the
 * carve-out rests on, and the reviewer will scrutinise it: the refresh token is opened ONLY on the
 * manager node, the control plane is handed only the ACCESS token in the clear (plus an opaque
 * re-sealed refresh envelope), and a consuming node writes an access-token-only credential and is
 * never delivered a refresh token — nowhere on the machine, nowhere in the control plane's database.
 *
 * The flow, driven deterministically (the runtime images are the real ones the host pulled; the
 * OAuth endpoint is a tiny node stub the test runs from the same image, so no vendor is reached):
 *   1. adopt: the manager runtime seals a refresh token at rest to a node key pair (the seal runs on
 *      the manager node; the plaintext never leaves it). The sealed envelope is stored via
 *      `licence set-grant` — the control plane stores ciphertext it cannot open.
 *   2. refresh: the manager runtime OPENS the envelope with the node's own key, calls the stub token
 *      endpoint, and writes out ONLY { access token, re-sealed refresh envelope }.
 *   3. submit: `licence submit-refresh` hands the control plane those two things — never the refresh
 *      token in the clear — and it seals the access token to the consumer holder.
 *   4. deliver: a push delivers the sealed access token to the consumer; the consumer runtime writes
 *      ~/.claude/.credentials.json, access-token-only.
 *
 * STUBBED, and flagged in the report: (a) the vendor OAuth endpoint (a node stub); (b) the manager
 * node's private sealing key, mounted as a test key pair — production needs a host capability to
 * place the node private key where a manager module reads it, which mesh-host does not have today;
 * (c) the submit transport (the test invokes `mesh-control licence submit-refresh` on the manager's
 * output, standing in for the authenticated cross-node call a manager node would make).
 *
 *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
 *   Build both runtime images into the local daemon first:
 *     scripts/build-module-runtime.sh anthropic-manager  /tmp/anthropic-manager.tar
 *     scripts/build-module-runtime.sh anthropic-consumer  /tmp/anthropic-consumer.tar
 */

import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { generateKeyPairSync } from "node:crypto";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";

const capability = await labIsUsable();
const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";

const skip = !capability.usable
  ? `lab not usable: ${capability.why}`
  : !binary || !existsSync(binary)
    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
    : !bundle || !existsSync(bundle)
      ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
      : false;

const SCENARIO = "anthropic-bed";
const MACHINE = "anchor";

// The fake tokens the flow moves. The whole test is: the second reaches the consumer, the first never
// does.
const REFRESH_TOKEN = "rt-lab-refresh-must-never-be-delivered";
const ACCESS_TOKEN = "at-lab-access-token-minted-by-the-stub";
const ROTATED_REFRESH = "rt-lab-rotated-still-must-never-be-delivered";

let instanceId = "";
let stocked: string[] = [];

function quote(s: string): string {
  return `'${s.replaceAll("'", `'\\''`)}'`;
}

async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
  const { stdout } = await exec(instanceId, MACHINE, [
    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
  ], timeoutMs);
  const marker = stdout.lastIndexOf("__exit=");
  if (marker < 0) return { out: stdout, ok: false };
  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
}

async function must(command: string, timeoutMs?: number): Promise<string> {
  const { out, ok } = await on(command, timeoutMs);
  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
  return out;
}

async function mesh(command: string, timeoutMs?: number): Promise<string> {
  return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
}

async function meshTry(command: string): Promise<{ out: string; ok: boolean }> {
  return on(`docker exec mesh-control /mesh-control ${command}`);
}

function pinned(repository: string): string {
  const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
  assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
  return found;
}

function bundleFor(images: string[]): string {
  let text = readFileSync(bundle, "utf8");
  for (const ref of images) {
    const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
    const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
    text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
  }
  return text;
}

function tokenFrom(said: string): string {
  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
  assert.ok(found, `no token in:\n${said}`);
  return found;
}

/** A node key pair as the mesh records it: raw 32-byte X25519 keys, standard base64. */
function nodeKeyPair(): { pub: string; priv: string } {
  const kp = generateKeyPairSync("x25519");
  const std = (b64url: string) => Buffer.from(b64url, "base64url").toString("base64");
  return {
    pub: std((kp.publicKey.export({ format: "jwk" }) as { x: string }).x),
    priv: std((kp.privateKey.export({ format: "jwk" }) as { d: string }).d),
  };
}

/** The local image id the host pulled for a runtime, so the test can drive it deterministically. */
async function imageId(substr: string): Promise<string> {
  const out = (await must(`docker images --no-trunc --format '{{.ID}} {{.Repository}}' | grep ${quote(substr)} | head -1`)).trim();
  const id = out.split(/\s+/)[0] ?? "";
  assert.ok(id.startsWith("sha256:") || id.length > 0, `no local image matched ${substr}:\n${out}`);
  return id;
}

async function settled(withinMs = 600_000): Promise<void> {
  const until = Date.now() + withinMs;
  let last = "";
  while (Date.now() < until) {
    const asked = await meshTry(`status --json`);
    if (asked.ok) {
      try {
        const state = JSON.parse(asked.out) as {
          wrong: { node: string; outcome: string }[];
          waiting: { node: string }[];
          reported: { node: string; outcome: string; current: boolean }[];
        };
        const bad = state.wrong.find((w) => w.node === MACHINE);
        if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
        const word = state.reported.find((r) => r.node === MACHINE);
        if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
        last = asked.out;
      } catch (err) {
        if (err instanceof Error && err.message.includes("did not apply")) throw err;
        last = asked.out;
      }
    }
    await new Promise((r) => setTimeout(r, 5000));
  }
  throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
}

before(async () => {
  if (skip) return;

  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
    onProgress: (m) => console.log(`raise: ${m}`),
  });
  instanceId = raised.instanceId;
  stocked = raised.images;

  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
  await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
  const up = await must(`docker ps --format '{{.Names}}'`);
  for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
    assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
  }

  await mesh(`node add ${MACHINE}`);
  const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
  await must(`${HOST_PATH} enrol --token ${quote(token)}`);
  await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
}, { timeout: 1_800_000 });

after(async () => {
  if (instanceId) await destroy(instanceId);
  await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });

test("model access refreshes on the manager node and delivers only the access token, never the refresh token", {
  skip, timeout: 1_500_000,
}, async () => {
  const managerImage = pinned("mesh-runtime-anthropic-manager");
  const consumerImage = pinned("mesh-runtime-anthropic-consumer");

  // --- the licence, its manager, and the consumer holder ------------------------------------------
  await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`);
  await mesh(`licence manager personal ${MACHINE}`);
  await mesh(`licence use personal ${MACHINE} anthropic-consumer`);

  // --- both runtimes are placed so the host pulls their images (which the test then drives) --------
  // Inline manifests, env pointed at the stub, mirroring the committed module.json. The scheduled
  // containers install as present state (ADR 0053); the test drives the entrypoints directly for a
  // deterministic flow rather than waiting on cron.
  const managerManifest = JSON.stringify({
    module: "anthropic-manager",
    version: "1",
    "own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" },
    emits: ["module.anthropic-manager.usage.read"],
    resources: [
      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" },
      { id: "keys", type: "directory", path: "/var/lib/mesh/anthropic-manager/keys", mode: "0700" },
      { id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" },
      {
        id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh",
        image: managerImage, network: "host", schedule: "*/9 * * * *",
        args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"],
        volumes: ["/var/lib/mesh/anthropic-manager:/run/state"],
        env: {
          MESH_ANTHROPIC_LICENCE: "personal",
          MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token",
          MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage",
          MESH_ANTHROPIC_GRANT_FILE: "/run/state/grant.json",
          MESH_NODE_SEALING_PUBLIC_FILE: "/run/state/keys/sealing.pub",
          MESH_NODE_SEALING_PRIVATE_FILE: "/run/state/keys/sealing.priv",
          MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token",
          MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json",
          MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json",
        },
      },
    ],
  });
  const consumerManifest = JSON.stringify({
    module: "anthropic-consumer",
    version: "1",
    requires: ["model-access"],
    binds: { "model-access": "/var/lib/anthropic-consumer/model.json" },
    secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" },
    "own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" },
    emits: ["module.anthropic-consumer.usage.session"],
    resources: [
      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" },
      { id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" },
      { id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" },
      {
        id: "apply", type: "container", name: "mesh-anthropic-consumer-apply",
        image: consumerImage, network: "host", schedule: "*/9 * * * *",
        args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"],
        volumes: ["/var/lib/anthropic-consumer:/run/state"],
        env: {
          MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token",
          MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json",
          MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json",
          MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json",
        },
      },
    ],
  });

  for (const [name, body] of [["anthropic-manager", managerManifest], ["anthropic-consumer", consumerManifest]] as const) {
    await must(`printf %s ${quote(body)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-control:/${name}.json`);
    await mesh(`module add /${name}.json`);
    await mesh(`module issue ${name} --node ${MACHINE}`);
    await mesh(`assign ${MACHINE} ${name}`);
  }
  await mesh(`push ${MACHINE}`);
  await settled();

  // The images the host pulled to run the scheduled containers are now local; drive them directly.
  const managerId = await imageId("anthropic-manager");
  const consumerId = await imageId("anthropic-consumer");

  // --- the stubbed node private key, mounted (FLAGGED) --------------------------------------------
  // Production needs a host capability to place the node private key where the manager reads it;
  // mesh-host has none today. Here the test mounts a generated key pair as that key.
  const keys = nodeKeyPair();
  await must(`printf %s ${quote(keys.pub)} > /var/lib/mesh/anthropic-manager/keys/sealing.pub`);
  await must(`printf %s ${quote(keys.priv)} > /var/lib/mesh/anthropic-manager/keys/sealing.priv`);

  // --- the OAuth stub: a tiny node server run from the manager image itself -----------------------
  const stub = [
    "const http=require('http');",
    "http.createServer((req,res)=>{let b='';req.on('data',c=>b+=c);req.on('end',()=>{",
    "  if(req.url.startsWith('/token')){res.setHeader('content-type','application/json');",
    `    res.end(JSON.stringify({access_token:${JSON.stringify(ACCESS_TOKEN)},refresh_token:${JSON.stringify(ROTATED_REFRESH)},expires_in:3600,refresh_token_expires_in:2592000,subscription_type:'pro'}));return;}`,
    "  if(req.url.startsWith('/usage')){res.setHeader('content-type','application/json');",
    "    res.end(JSON.stringify({five_hour:{utilization:12,resets_at:'2026-01-01T00:00:00Z'},seven_day:{utilization:3}}));return;}",
    "  res.statusCode=404;res.end('no');});}).listen(9099,'127.0.0.1',()=>console.log('stub up'));",
  ].join("\n");
  await must(`printf %s ${quote(stub)} > /var/lib/mesh/anthropic-manager/stub.js`);
  await must(`docker rm -f oauth-stub 2>/dev/null; docker run -d --name oauth-stub --network host --entrypoint node -v /var/lib/mesh/anthropic-manager/stub.js:/run/stub.js:ro ${managerId} /run/stub.js`);
  // Give it a moment to bind.
  await must(`for i in $(seq 1 20); do curl -s -X POST http://127.0.0.1:9099/token >/dev/null && break; sleep 1; done`);

  // --- 1. adopt: seal the refresh token at rest, on the manager node ------------------------------
  await must(`printf %s ${quote(REFRESH_TOKEN)} > /var/lib/mesh/anthropic-manager/refresh-token`);
  await must(
    `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` +
      `-e MESH_ANTHROPIC_REFRESH_TOKEN_FILE=/run/state/refresh-token ` +
      `-e MESH_NODE_SEALING_PUBLIC_FILE=/run/state/keys/sealing.pub ` +
      `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/grant.json ` +
      `${managerId} run /app/modules/anthropic-manager/dist/adopt/index.js`,
  );
  const envelope = await must(`cat /var/lib/mesh/anthropic-manager/grant.json`);
  assert.doesNotMatch(envelope, new RegExp(REFRESH_TOKEN),
    `the adopted envelope holds the refresh token in the clear:\n${envelope}`);

  // Store the sealed envelope in the control plane — which never sees the refresh token.
  await must(`docker cp /var/lib/mesh/anthropic-manager/grant.json mesh-control:/grant.json`);
  await mesh(`licence set-grant personal --file /grant.json`);

  // --- 2. refresh: open on the manager node, call the stub, write access token + re-sealed refresh -
  await must(
    `docker run --rm --network host -v /var/lib/mesh/anthropic-manager:/run/state ` +
      `-e MESH_ANTHROPIC_LICENCE=personal ` +
      `-e MESH_ANTHROPIC_TOKEN_ENDPOINT=http://127.0.0.1:9099/token ` +
      `-e MESH_ANTHROPIC_USAGE_ENDPOINT=http://127.0.0.1:9099/usage ` +
      `-e MESH_ANTHROPIC_GRANT_FILE=/run/state/grant.json ` +
      `-e MESH_NODE_SEALING_PUBLIC_FILE=/run/state/keys/sealing.pub ` +
      `-e MESH_NODE_SEALING_PRIVATE_FILE=/run/state/keys/sealing.priv ` +
      `-e MESH_ANTHROPIC_ACCESS_OUT=/run/state/out/access-token ` +
      `-e MESH_ANTHROPIC_GRANT_OUT=/run/state/out/grant.json ` +
      `-e MESH_ANTHROPIC_USAGE_OUT=/run/state/out/usage.json ` +
      `${managerId} run /app/modules/anthropic-manager/dist/refresh/index.js`,
  );
  const producedAccess = (await must(`cat /var/lib/mesh/anthropic-manager/out/access-token`)).trim();
  assert.equal(producedAccess, ACCESS_TOKEN, "the manager did not mint the stub's access token");
  const newEnvelope = await must(`cat /var/lib/mesh/anthropic-manager/out/grant.json`);
  assert.doesNotMatch(newEnvelope, new RegExp(ROTATED_REFRESH),
    `the re-sealed envelope holds the rotated refresh token in the clear:\n${newEnvelope}`);
  // Licence-grain usage was read and recorded.
  const usage = await must(`cat /var/lib/mesh/anthropic-manager/out/usage.json`);
  assert.match(usage, /"sessionPct":12/, `the licence-grain usage reading is wrong:\n${usage}`);

  // --- 3. submit: the control plane is handed only the access token + opaque envelope -------------
  await must(`docker cp /var/lib/mesh/anthropic-manager/out/access-token mesh-control:/access-token`);
  await must(`docker cp /var/lib/mesh/anthropic-manager/out/grant.json mesh-control:/new-grant.json`);
  const submitted = await mesh(`licence submit-refresh personal --access-file /access-token --grant-file /new-grant.json`);
  assert.match(submitted, /sealed to 1 holder/, submitted);

  // --- 4. deliver: the consumer gets the sealed access token and writes the credential -------------
  await mesh(`push ${MACHINE}`);
  await settled();

  // Drive the consumer's apply once the token has been delivered to its secret path.
  let delivered = false;
  for (let i = 0; i < 20 && !delivered; i++) {
    delivered = (await on(`test -s /var/lib/anthropic-consumer/access-token`)).ok;
    if (!delivered) await new Promise((r) => setTimeout(r, 3000));
  }
  assert.ok(delivered, "the sealed access token was never delivered to the consumer's secret path");
  await must(
    `docker run --rm --network host -v /var/lib/anthropic-consumer:/run/state ` +
      `-e MESH_MODEL_ACCESS_SECRET_FILE=/run/state/access-token ` +
      `-e MESH_MODEL_ACCESS_BIND_FILE=/run/state/model.json ` +
      `-e MESH_CLAUDE_CREDENTIALS_FILE=/run/state/claude/.credentials.json ` +
      `-e MESH_CLAUDE_IDENTITY_FILE=/run/state/claude/.claude.json ` +
      `${consumerId} run /app/modules/anthropic-consumer/dist/apply/index.js`,
  );

  // --- the invariant, asserted from every angle ---------------------------------------------------
  const creds = await must(`cat /var/lib/anthropic-consumer/claude/.credentials.json`);
  assert.match(creds, new RegExp(ACCESS_TOKEN), `the access token did not reach the credential file:\n${creds}`);
  const parsed = JSON.parse(creds) as { claudeAiOauth?: { accessToken?: string; refreshToken?: string } };
  assert.equal(parsed.claudeAiOauth?.accessToken, ACCESS_TOKEN);
  assert.ok(!parsed.claudeAiOauth?.refreshToken, "the consumer was given a refresh token");

  // The refresh token — original or rotated — is nowhere on the consuming node.
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    const found = await on(`grep -rq ${quote(secret)} /var/lib/anthropic-consumer`);
    assert.ok(!found.ok, `a refresh token is on the consuming node under /var/lib/anthropic-consumer`);
  }

  // The control plane's own database holds the refresh token only as ciphertext.
  const grantRow = await must(
    `docker exec mesh-store psql -U postgres -d licences -qAt -c "select token, wrapped_key from refresh_grant where licence='personal'"`,
  );
  assert.ok(grantRow.trim().length > 0, "no refresh grant was stored");
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    assert.doesNotMatch(grantRow, new RegExp(secret),
      `the refresh token is in the control plane's database in the clear:\n${grantRow}`);
  }
  // And the holder's sealed column carries the access token's seal, never a refresh token.
  const holder = await must(
    `docker exec mesh-store psql -U postgres -d licences -qAt -c "select coalesce(sealed,'') from licence_holder where licence='personal'"`,
  );
  assert.ok(holder.trim().length > 0, "nothing was sealed to the holder");
  for (const secret of [REFRESH_TOKEN, ROTATED_REFRESH]) {
    assert.doesNotMatch(holder, new RegExp(secret), "a refresh token is in the holder row");
  }

  await must(`docker rm -f oauth-stub 2>/dev/null || true`);
});