A public name, against a real ACME server

The other half of the certificate split: the mesh's own authority
certifies internal names, and a name reachable from outside needs one
the world already trusts. Against a real server rather than a stub,
because what is under test is whether an order, a challenge and a
handshake agree, and a stub would be told to agree.

One assertion passes and one fails, and the failure is filed as
novox/hq 04-ISSUES/020: the authority issues a certificate and the
client never collects it. Kept as a failing test rather than deleted or
skipped — it is the reproduction, and it proves everything up to the
last hop.

The passing one is the guard that matters day to day: no certificate is
ordered for a name the mesh does not route, so a scan cannot spend an
account's rate limit.

The failure output gathers both sides before asserting. The first
version reported only what the proxy said, which made a server-side
question unanswerable — "the client never spoke to it" and "it refused
what the client said" are different faults with nothing in common.

scenarios/a-public-name.yml

@@ -0,0 +1,26 @@
+# One machine serving a public name with a certificate from an authority it did not run itself.
+#
+# The lab keeps production's two-authority split rather than collapsing it (01-RESEARCH/004): the
+# mesh's own authority certifies `.internal` names, and a name reachable from outside is certified
+# by ACME. A single-authority lab would hide any fault living in that split, so this raises a real
+# ACME server and makes the proxy actually order from it.
+#
+# Pebble rather than a stub, for the reason the lab exists at all: what is under test is whether an
+# HTTP-01 challenge is answered at the name being certified, and a fake would be told to agree.
+scenario: a-public-name
+
+segments:
+  hosting:
+    kind: public
+    cidr: [192.0.2.0/24]
+
+machines:
+  anchor:
+    at: { segment: hosting, address: [192.0.2.10] }
+    inbound: allow
+
+images:
+  - ghcr.io/letsencrypt/pebble:2.5.0
+
+place:
+  all: [runtime]

test/integration/certificates.test.ts

@@ -0,0 +1,191 @@
+/**
+ * A public name, served with a certificate from an authority the mesh did not run.
+ *
+ * The mesh's own authority certifies `.internal` names and is proven elsewhere. This is the other
+ * half of the split: a name reachable from outside needs a certificate somebody else's browser
+ * already trusts, which means ordering one over ACME and answering a challenge **at the name being
+ * certified**.
+ *
+ * Against a real ACME server rather than a stub, for the reason the lab exists: what is under test
+ * is whether an order, a challenge and a handshake agree with each other, and a stub would be told
+ * to agree.
+ */
+
+import { test, after, before } from "node:test";
+import assert from "node:assert/strict";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { labIsUsable, destroyAll } from "./harness.ts";
+import { incus } from "../../src/incus/client.ts";
+import { machineName } from "../../src/lifecycle/names.ts";
+
+const capability = await labIsUsable();
+const proxy = process.env["MESH_LAB_ROUTE_PROXY"] ?? "";
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !proxy
+    ? "set MESH_LAB_ROUTE_PROXY to a built proxy (mesh-control: go build ./examples/route-proxy)"
+    : false;
+
+const SCENARIO = "a-public-name";
+const MACHINE = "anchor";
+const NAME = "photos.example";
+const ACME = "/var/lib/acme";
+
+let instanceId = "";
+
+function shellQuote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(command: string): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, MACHINE, [
+    "sh", "-c", `${command} 2>&1; echo "__exit=$?"`,
+  ]);
+  const marker = stdout.lastIndexOf("__exit=");
+  return { out: stdout.slice(0, marker), ok: Number(stdout.slice(marker + 7).trim()) === 0 };
+}
+
+async function must(command: string): Promise<string> {
+  const { out, ok } = await on(command);
+  if (!ok) throw new Error(`${command}\n${out}`);
+  return out;
+}
+
+before(async () => {
+  if (skip) return;
+  const scenario = loadScenario(`scenarios/${SCENARIO}.yml`);
+  const instance = await raise(scenario, {});
+  instanceId = instance.instanceId;
+
+  const pebble = instance.images.find((r) => r.includes("pebble"));
+  assert.ok(pebble, `the scenario stocked no ACME server: ${instance.images.join(", ")}`);
+
+  await must(`mkdir -p ${ACME}/cache`);
+
+  // The authority's own API certificate is signed by a root nothing trusts yet. Taken out of the
+  // image rather than disabling verification, which is the same reason the proxy names a bundle:
+  // "skip" would still apply on the day this points at a public authority.
+  await must(`docker create --name pebble-certs ${pebble}`);
+  await must(`docker cp pebble-certs:/test/certs/pebble.minica.pem ${ACME}/authority-api.pem`);
+  await must(`docker rm pebble-certs`);
+
+  // **The challenge must arrive on port 80**, which is where a proxy serving a public name
+  // listens. The authority's own default is 5002 — convenient for its test suite and wrong here,
+  // because the thing being proven is that the real path works.
+  //
+  // **Its own configuration, with one field changed.** The first version of this wrote a config
+  // from scratch and silently dropped two fields the default carries; the order then came back
+  // valid with no certificate to fetch, and the failure looked like a client bug. Take what works
+  // and change the one thing that must differ.
+  await must(`docker create --name pebble-config ${pebble}`);
+  await must(`docker cp pebble-config:/test/config/pebble-config.json ${ACME}/pebble.json`);
+  await must(`docker rm pebble-config`);
+  await must(
+    `python3 -c "import json,sys;` +
+      `c=json.load(open('${ACME}/pebble.json'));` +
+      `c['pebble']['httpPort']=80;` +
+      `json.dump(c,open('${ACME}/pebble.json','w'),indent=2)"`,
+  );
+
+  // The name resolves to this machine, so the authority's challenge reaches the proxy rather than
+  // whatever else on the internet answers to it.
+  await must(`grep -q ${shellQuote(NAME)} /etc/hosts || echo "127.0.0.1 ${NAME}" >> /etc/hosts`);
+
+  await must(
+    `docker run -d --name acme --network host ` +
+      `-v ${ACME}/pebble.json:/test/config/pebble-config.json:ro ` +
+      `${pebble} -config /test/config/pebble-config.json -dnsserver 127.0.0.53:53`,
+  );
+
+  let up = false;
+  for (let i = 0; i < 60 && !up; i++) {
+    ({ ok: up } = await on(
+      `curl -sf --cacert ${ACME}/authority-api.pem https://127.0.0.1:14000/dir -o /dev/null`,
+    ));
+    if (!up) await new Promise((r) => setTimeout(r, 1000));
+  }
+  assert.ok(up, `the ACME server never answered:\n${(await on(`docker logs acme`)).out}`);
+
+  await incus([
+    "file", "push", proxy,
+    `${machineName(instanceId, MACHINE)}/usr/local/bin/mesh-route-proxy`,
+    "--mode", "0755",
+  ], 180_000);
+
+  // Something for the route to point at, so the proxy is serving a real name and not a hole.
+  await must(
+    `printf %s ${shellQuote(JSON.stringify({
+      given: [{ from: "photos", node: "", at: "", values: { name: NAME, port: 8080 } }],
+    }))} > ${ACME}/routes.json`,
+  );
+  await must(
+    `nohup sh -c 'while true; do printf "HTTP/1.1 200 OK\\r\\nContent-Length: 5\\r\\n\\r\\nhello" | nc -l -p 8080 -q 1; done' >/dev/null 2>&1 &`,
+  );
+}, { timeout: 1_200_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("a public name is served with a certificate the mesh did not issue", {
+  skip, timeout: 600_000,
+}, async () => {
+  await must(
+    `ROUTES=${ACME}/routes.json LISTEN=:80 TLS_LISTEN=:443 ` +
+      `ACME_CACHE=${ACME}/cache ` +
+      `ACME_DIRECTORY=https://127.0.0.1:14000/dir ` +
+      `ACME_CA_BUNDLE=${ACME}/authority-api.pem ` +
+      `nohup /usr/local/bin/mesh-route-proxy >${ACME}/proxy.log 2>&1 & sleep 3`,
+  );
+
+  // The authority's issuing root, so the handshake can be checked rather than merely completed.
+  await must(
+    `curl -sf --cacert ${ACME}/authority-api.pem https://127.0.0.1:15000/roots/0 > ${ACME}/issuer.pem`,
+  );
+
+  // The first request is what triggers the order: autocert obtains on demand for a name its
+  // policy allows. Retried because ordering, the challenge and issuance take a moment.
+  let served = { out: "", ok: false };
+  for (let i = 0; i < 40 && !served.ok; i++) {
+    served = await on(`curl -sf --cacert ${ACME}/issuer.pem https://${NAME}/ `);
+    if (!served.ok) await new Promise((r) => setTimeout(r, 2000));
+  }
+  if (!served.ok) {
+    // Both sides, gathered before asserting. The proxy's log says what it tried; the authority's
+    // says whether it ever heard from it — and "the client never spoke to it" and "it refused
+    // what the client said" are different faults with nothing in common.
+    const proxyLog = (await on(`cat ${ACME}/proxy.log`)).out;
+    const authority = (await on(`docker logs acme 2>&1 | tail -40`)).out;
+    const directory = (await on(
+      `curl -s --cacert ${ACME}/authority-api.pem https://127.0.0.1:14000/dir`)).out;
+    assert.fail(
+      `the name was never served over TLS: ${served.out}\n\n` +
+      `── the proxy tried:\n${proxyLog}\n` +
+      `── the authority heard:\n${authority}\n` +
+      `── the directory it was pointed at:\n${directory}\n`);
+  }
+  assert.match(served.out, /hello/);
+
+  // And it is the authority's certificate, not something self-signed that happens to work.
+  const issuer = await must(
+    `echo | openssl s_client -connect ${NAME}:443 -servername ${NAME} 2>/dev/null ` +
+      `| openssl x509 -noout -issuer -subject`,
+  );
+  assert.match(issuer, /Pebble/i, `the certificate was not issued by the ACME server:\n${issuer}`);
+  assert.match(issuer, new RegExp(NAME), `the certificate is not for the name asked for:\n${issuer}`);
+});
+
+test("no certificate is ordered for a name the mesh does not route", {
+  skip, timeout: 300_000,
+}, async () => {
+  // The policy that stops a quota being spent by a scan. Refused before any order is placed, so
+  // the authority never sees it.
+  const { out } = await on(
+    `echo | openssl s_client -connect 127.0.0.1:443 -servername nobody-asked-for-this.example 2>&1 | head -20`,
+  );
+  assert.doesNotMatch(out, /Pebble/i,
+    `a certificate was obtained for a name nothing routes here:\n${out}`);
+});