From 6591a2e0405017ac3b0a1d179093a7e3bc6c450c Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 1 Sep 2026 16:24:19 +0200 Subject: [PATCH] A canary first: one machine, one path, three minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suggested by Jochen, and it paid for itself on its first run. A suite that takes forty minutes is a suite you hear from once a day. Every fault found today would have shown up in the first three minutes of it — a module pinned to an image that does not exist, a consumer given a password and no name to present with it, a credential file nothing could read, a search for a password that read the password as an option. The other thirty-seven minutes proved things that were already working. So this runs first, on one machine, with the three images the mesh needs for itself. It walks one path: a mesh comes up, a module lands, and a consumer gets a credential it can actually use — the name to present, the address, the port, and a password only the host could put there. Deliberately not a smaller copy of the full suite: that path is where everything went wrong, and a canary checking many things shallowly is a canary whose failure nobody can read. `suite` runs it and stops if it dies, saying why rather than leaving somebody to wonder what the missing thirty-seven minutes would have said. Skipped when the caller named its own files. It measured 164 seconds against forty-odd minutes, and failed three times on its first run for one reason: applying the bundle raises a control plane but does not tell it a machine exists. I had left out enrolment, and the long suite would have taken forty minutes to say so. --- src/lastrun.ts | 9 ++ src/suite.ts | 32 +++++- test/integration/canary.test.ts | 177 ++++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 test/integration/canary.test.ts diff --git a/src/lastrun.ts b/src/lastrun.ts index 9d76fe4..48f3893 100644 --- a/src/lastrun.ts +++ b/src/lastrun.ts @@ -50,6 +50,15 @@ export interface Receipt { */ export const endToEnd = "test/integration/mesh.test.ts"; +/** + * canary is the short run that goes first. + * + * One machine, three images, one path walked end to end. **A suite that takes forty minutes is a + * suite you hear from once a day** — and every fault found on 2026-09-01 would have shown up in + * the first three minutes of it. Running this first means a broken change costs minutes. + */ +export const canary = "test/integration/canary.test.ts"; + /** Where the receipt lives: XDG state, which is for exactly this — data a tool keeps between runs. */ export function receiptPath(): string { const state = process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"); diff --git a/src/suite.ts b/src/suite.ts index 8444c8d..96fdafa 100644 --- a/src/suite.ts +++ b/src/suite.ts @@ -10,7 +10,7 @@ */ import { spawn } from "node:child_process"; -import { endToEnd, record } from "./lastrun.ts"; +import { canary, endToEnd, record } from "./lastrun.ts"; import { rebuild } from "./rebuild.ts"; /** counted is what the runner said, or nulls when it said nothing recognisable. */ @@ -44,6 +44,32 @@ export async function runSuite(args: string[]): Promise { if (built.length > 0) console.log(`built: ${built.join(", ")}\n`); } + // **The canary first, and stop if it dies.** It walks one path on one machine: a mesh comes up, + // a module lands, a consumer gets a credential it can use. Everything that broke on + // 2026-09-01 broke on that path, and finding out took forty minutes each time because the long + // run had to reach it. + // + // Skipped when the caller named its own files — they asked for something specific — and when + // the canary is itself what was asked for. + if (ran.length === 0) { + const first = await runFiles([canary]); + if (first.code !== 0) { + console.log( + `\nthe canary failed, so the rest was not run. It is one machine and one path: a mesh ` + + `comes up, a module lands, a consumer gets a credential. Fix that first — the long ` + + `suite would fail on the same thing forty minutes later.`); + return first.code; + } + console.log(""); + } + + const { code, seen } = await runFiles(files); + console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files))); + return code; +} + +/** Run some test files, passing their output through as it arrives. */ +async function runFiles(files: string[]): Promise<{ code: number; seen: string }> { const running = spawn( process.execPath, ["--test", "--test-concurrency=1", "--experimental-strip-types", @@ -61,9 +87,7 @@ export async function runSuite(args: string[]): Promise { const code: number = await new Promise((resolve) => { running.on("close", (c) => resolve(c ?? 1)); }); - - console.log("\n" + reportOn(counted(seen), (p, f) => record(p, f, files))); - return code; + return { code, seen }; } /** diff --git a/test/integration/canary.test.ts b/test/integration/canary.test.ts new file mode 100644 index 0000000..7584f22 --- /dev/null +++ b/test/integration/canary.test.ts @@ -0,0 +1,177 @@ +/** + * The shortest run that would have caught today's faults. + * + * **A suite that takes forty minutes is a suite you find out from once a day.** Every fault found + * on 2026-09-01 — a module pinned to an image that does not exist, a consumer given a password and + * no name to present with it, a credential file nothing could read, a search for a password that + * read the password as an option — would have shown up in the first three minutes of it. The other + * thirty-seven proved things that were already working. + * + * So this runs first, on one machine, with the three images the mesh needs for itself and nothing + * else. If it fails there is no point spending the rest. + * + * It is deliberately *not* a smaller copy of the full suite. It walks one path end to end — a mesh + * comes up, a module reaches a machine, and a consumer gets a credential it can actually use — + * because that path is where everything went wrong, and a canary that checks many things shallowly + * is a canary nobody can read the failure of. + */ + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readFileSync } from "node:fs"; + +import { raise } from "../../src/lifecycle/raise.ts"; +import { exec, destroy } from "../../src/lifecycle/operate.ts"; +import { loadScenario } from "../../src/declaration/parse.ts"; +import { labIsUsable } from "./harness.ts"; +import { pinnedInto } from "../../src/pinning.ts"; + +const capability = await labIsUsable(); +const host = process.env["MESH_LAB_HOST_BINARY"] ?? ""; +const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; + +const skip = !capability.usable + ? `lab not usable: ${capability.why}` + : !host || !existsSync(host) + ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" + : !bundle || !existsSync(bundle) + ? "MESH_LAB_BUNDLE is not set to a substrate bundle" + : false; + +const SCENARIO = "first-node"; +const MACHINE = "anchor"; +const HOST_PATH = "/usr/local/bin/mesh-host"; +let instanceId = ""; + +function quote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> { + const { stdout } = await exec(instanceId, MACHINE, [ + "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`, + ], timeoutMs); + const marker = stdout.lastIndexOf("__exit="); + return { out: stdout.slice(0, marker), ok: Number(stdout.slice(marker + 7).trim()) === 0 }; +} + +async function must(command: string, timeoutMs?: number): Promise { + const { out, ok } = await on(command, timeoutMs); + if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`); + return out; +} + +const mesh = (command: string, timeoutMs?: number) => + must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs); + +before(async () => { + if (skip) return; + const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { + onProgress: (m) => console.log(`raise: ${m}`), + }); + instanceId = raised.instanceId; + await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${ + pinnedInto(readFileSync(bundle, "utf8"), raised.images)}\nMESHBUNDLE`); + await must(`${HOST_PATH} apply /tmp/substrate.lock`, 300_000); + + // **And the machine joins.** Applying the bundle raises a control plane; it does not tell that + // control plane a machine exists. Leaving this out is what the first run of this canary found, + // in under three minutes: the mesh answered, said "0 machine(s)", and every assignment after it + // failed with `no node of that name`. + await mesh(`node add ${MACHINE}`); + const said = await mesh(`token issue --node ${MACHINE}`); + const token = said.split("\n").map((l) => l.trim()) + .find((l) => l.length > 100 && !l.includes(" ")); + assert.ok(token, `no token in:\n${said}`); + await must(`${HOST_PATH} enrol --token ${quote(token)}`); + + // The host has to be running for a push to reach it. + await must(`pgrep -x mesh-host >/dev/null || ` + + `(setsid nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 < /dev/null & sleep 3)`); +}); + +after(async () => { + // The canary owns its scenario and takes it down. Left standing it would hold a machine for + // the forty minutes of the run it exists to protect. + if (instanceId) await destroy(instanceId).catch(() => {}); +}); + +test("a mesh comes up and answers", { skip, timeout: 600_000 }, async () => { + const said = await mesh("status"); + assert.match(said, /anchor/, `the mesh does not know the machine it is running on:\n${said}`); +}); + +// One module, no images, nothing to stock. What is under test is the chain — added, assigned, +// resolved, planned, pushed, applied, reported — not what is at the end of it. +test("a module reaches the machine", { skip, timeout: 600_000 }, async () => { + await must(`printf %s ${quote(JSON.stringify({ + module: "canary", version: "1", + resources: [ + { id: "state", type: "directory", path: "/var/lib/canary", mode: "0700" }, + { id: "note", type: "file", path: "/var/lib/canary/it-arrived", content: "yes\n", mode: "0644" }, + ], + }))} > /tmp/canary.json`); + await must(`docker cp /tmp/canary.json mesh-control:/canary.json`); + await mesh("module add /canary.json"); + await mesh(`assign ${MACHINE} canary`); + await mesh(`push ${MACHINE}`, 300_000); + + assert.equal((await must(`cat /var/lib/canary/it-arrived`)).trim(), "yes"); + assert.match(await must(`stat -c %a /var/lib/canary`), /^700/); +}); + +// **The half that broke all day.** A provider and a consumer on one machine: the mesh makes a +// credential, tells the provider who asked, and gives the consumer a file it can read — with the +// name to present, which it could not have known (novox/hq 04-ISSUES/021, 022, 023). +test("a consumer gets a credential it can use", { skip, timeout: 600_000 }, async () => { + await must(`printf %s ${quote(JSON.stringify({ + module: "canary-store", version: "1", + provides: [{ name: "canary-database", scope: "mesh" }], + serves: { "canary-database": { port: 5432 } }, + receives: { "canary-database": "/var/lib/canary-store/asked.json" }, + grants: { "canary-database": "/var/lib/canary-store/grants" }, + resources: [ + { id: "state", type: "directory", path: "/var/lib/canary-store", mode: "0700" }, + { id: "grants", type: "directory", path: "/var/lib/canary-store/grants", mode: "0700" }, + ], + }))} > /tmp/canary-store.json`); + await must(`printf %s ${quote(JSON.stringify({ + module: "canary-app", version: "1", + requires: ["canary-database"], + contributes: { "canary-database": { name: "canaryapp" } }, + binds: { "canary-database": "/var/lib/canary-app/where.json" }, + secrets: { "canary-database": "/var/lib/canary-app/password" }, + resources: [ + { id: "state", type: "directory", path: "/var/lib/canary-app", mode: "0700" }, + { + id: "env", type: "file", path: "/var/lib/canary-app/database.env", mode: "0600", + content: "PGHOST=${bound:canary-database:at}\nPGPORT=${bound:canary-database:port}\n" + + "PGUSER=${bound:canary-database:as}\nPGPASSWORD=${secret:canary-database}\n", + }, + ], + }))} > /tmp/canary-app.json`); + for (const name of ["canary-store", "canary-app"]) { + await must(`docker cp /tmp/${name}.json mesh-control:/${name}.json`); + await mesh(`module add /${name}.json`); + await mesh(`assign ${MACHINE} ${name}`); + } + await mesh(`push ${MACHINE}`, 300_000); + + const password = (await must(`cat /var/lib/canary-app/password`)).trim(); + assert.ok(password.length >= 40, `the consumer's credential is ${password.length} characters`); + + // Every hole filled, and filled with the right thing. + const env = await must(`cat /var/lib/canary-app/database.env`); + assert.match(env, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${env}`); + assert.match(env, /^PGUSER=mesh_[a-z0-9_]+_canary_app$/m, + `the consumer was not told what name to present:\n${env}`); + assert.doesNotMatch(env, /\$\{/, `a placeholder reached the machine as a value:\n${env}`); + assert.ok(env.includes(`PGPASSWORD=${password}`), + `the file holds a different password from the credential file:\n${env}`); + + // And the provider was told who asked, which is what makes the credential real. + const asked = await must(`cat /var/lib/canary-store/asked.json`); + assert.match(asked, /canary-app/, `the provider was not told who asked:\n${asked}`); + assert.match(asked, /"as": "mesh_[a-z0-9_]+_canary_app"/, + `the provider was not told what to call the login:\n${asked}`); +});