Unify trunk on main: initialization → main #3

Merged
jschoubben merged 95 commits from initialization into main 2026-09-05 01:13:46 +00:00
5 changed files with 490 additions and 8 deletions
Showing only changes of commit a516ee847b - Show all commits
+4
View File
@@ -28,6 +28,10 @@ images:
# what the mesh's registry is built from — the same chicken-and-egg the bootstrap has, resolved
# the same way.
- registry:2
# A real third-party workload, for adopting one the way the conversion will. Its database is
# the substrate's postgres image rather than its own: what is under test is the mesh delivering
# a module, not which postgres it delivers.
- ghcr.io/umami-software/umami:postgresql-latest
# And the builder, because it is a module the mesh assigns rather than a program somebody
# starts by hand — which is the only way its credential can be one the mesh delivered.
- mesh-builder:development
+35
View File
@@ -30,6 +30,8 @@ const USAGE = `mesh-lab — raise a disposable mesh on one machine
diagram <scenario.yml> [out.drawio] draw what a scenario asks for
diagram --live <instance> [out.drawio] draw what is actually raised
warm the scenario kept between runs, and whether it still counts
warm cool destroy it and forget it
suite [paths...] [--no-build] rebuild the artifacts, run the end-to-end tests, leave a receipt
last-run whether the last run still counts; non-zero when it does not
@@ -118,6 +120,39 @@ async function main(): Promise<void> {
return;
}
// A base state many tests start from, rather than each raising its own mesh.
//
// **The speed is the lesser half.** Tests that share one long-lived mesh accumulate each
// other's state, and a test that reads what the previous one left is a test that passes for
// the wrong reason — which has already happened here once. Returning to a named state between
// tests makes each of them independent.
case "warm": {
const { remembered, ready, cool } = await import("./warm.ts");
const what = rest[0] ?? "status";
if (what === "cool") {
const gone = await cool();
console.log(gone ? `destroyed ${gone}, and forgot it` : "nothing was being kept warm");
return;
}
const held = remembered();
if (!held) {
console.log("nothing is being kept warm.");
console.log(" a scenario is warmed by whatever brought it to a state worth keeping;");
console.log(" the integration suite does it when MESH_LAB_WARM is set.");
return;
}
console.log(`${held.instanceId} — ${held.scenario}, warmed ${held.at}`);
for (const [name, commit] of Object.entries(held.against)) {
console.log(` ${name.padEnd(14)} ${commit}`);
}
const said = await ready(held.scenario);
console.log(said.use === "restore"
? "\n usable: it can be returned to"
: `\n NOT usable: ${said.why}`);
if (said.use !== "restore") process.exitCode = 1;
return;
}
case "base": {
// `base build` exists because a sealed scenario cannot install a container runtime, and
// the runtime has to come from somewhere with a network (novox/hq ADR 0006).
+210
View File
@@ -0,0 +1,210 @@
/**
* A scenario kept between runs, already brought to a state worth starting from.
*
* **Bootstrapping a mesh takes minutes and proves the same thing every time.** The tests worth
* iterating on are the ones after it — assigning a module, adopting a workload, watching something
* fail. A warm instance is raised once, brought to that state, snapshotted, and restored on every
* later run in seconds.
*
* **The danger is precisely the one 04-ISSUES/005 is about**, one level down: a mesh snapshotted
* against yesterday's binaries will pass today's tests and report green, and nothing about the
* result would say what it was actually run against. So a warm instance records the commits it was
* built from, and is refused — not silently rebuilt, refused — when they have moved.
*
* **Fresh stays the default.** This is for iterating. A run that is meant to mean something raises
* from nothing, because "it passes" must not quietly come to mean "it passes against a mesh
* somebody bootstrapped last week".
*/
import { readFileSync, writeFileSync, mkdirSync, rmSync } from "node:fs";
import { dirname, join } from "node:path";
import { homedir } from "node:os";
import { list, restore, snapshot, snapshots, destroy } from "./lifecycle/operate.ts";
import type { Against } from "./lastrun.ts";
import { whatWasTested } from "./lastrun.ts";
/** The state a warm instance is kept at. One label, because a second is a state nobody named. */
export const label = "warm";
export interface Warm {
scenario: string;
instanceId: string;
/**
* The image references the scenario's registry serves, pinned by digest.
*
* Kept because they are worked out while raising and a restored instance never raises. Without
* them a warm run knows nothing about what it can pull, and every test naming an image fails
* for a reason that has nothing to do with what it was testing.
*/
images: string[];
/** The commit each repository was at when this was brought to its state. */
against: Against;
at: string;
}
/** Where the record lives: XDG state, beside the run receipt, for the same reason. */
export function recordPath(): string {
const state = process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state");
return join(state, "mesh-lab", "warm.json");
}
export function remember(warm: Warm): void {
const path = recordPath();
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, JSON.stringify(warm, null, 2) + "\n");
}
export function remembered(): Warm | null {
try {
return JSON.parse(readFileSync(recordPath(), "utf8")) as Warm;
} catch {
return null;
}
}
export function forget(): void {
rmSync(recordPath(), { force: true });
}
export type Verdict =
| { use: "restore"; instanceId: string }
| { use: "raise"; why: string };
/**
* judge decides whether a remembered instance may be restored.
*
* **Every reason to refuse is a reason a test would otherwise pass while meaning nothing**, so
* each is named rather than collapsed into "not usable".
*/
export function judge(
warm: Warm | null,
scenario: string,
standing: string[],
hasSnapshot: boolean,
against: Against,
): Verdict {
if (!warm) return { use: "raise", why: "nothing is being kept warm" };
if (warm.scenario !== scenario) {
return { use: "raise", why: `what is kept warm is ${warm.scenario}, and this is ${scenario}` };
}
if (!standing.includes(warm.instanceId)) {
return { use: "raise", why: `${warm.instanceId} is no longer standing` };
}
if (!hasSnapshot) {
return { use: "raise", why: `${warm.instanceId} has no ${label} snapshot to return to` };
}
// The check that keeps this honest. A mesh built from code that has since moved would pass
// today's tests against yesterday's binaries, and say nothing about it.
//
// **Both directions, because comparing only what is in front of you clears what is not.** The
// first version walked the current repositories alone, so running without the environment that
// names where they are compared nothing and reported the mesh usable — a warm instance built
// from code that had since moved, cleared by a check that had looked at neither. That is
// 04-ISSUES/005's rule again: a record that says nothing about something is not a record that
// clears it.
for (const name of new Set([...Object.keys(warm.against), ...Object.keys(against)])) {
const then = warm.against[name];
const now = against[name];
if (then === now) continue;
if (!now) {
return {
use: "raise",
why: `${name} was at ${then} when this was warmed, and nothing says where it is now — ` +
`so nothing can say whether it moved`,
};
}
return {
use: "raise",
why: `${name} was at ${then ?? "nothing recorded"} when this was warmed, ` +
`and is now at ${now}`,
};
}
return { use: "restore", instanceId: warm.instanceId };
}
/** What is standing right now, by instance. */
export async function standingNow(): Promise<string[]> {
return (await list()).map((i) => i.instanceId);
}
/**
* ready returns an instance already at its warm state, or says why one must be raised.
*
* It never raises: raising needs a scenario, images and a bootstrap, and all of that belongs to
* whoever is using this rather than here.
*/
export async function ready(
scenario: string,
env: NodeJS.ProcessEnv = process.env,
): Promise<Verdict> {
const warm = remembered();
const standing = await standingNow();
const has = warm ? (await snapshots(warm.instanceId)).includes(label) : false;
return judge(warm, scenario, standing, has, whatWasTested(env));
}
/** returnTo puts a warm instance back to its state, and says how long it took. */
export async function returnTo(
instanceId: string,
log: (message: string) => void = () => {},
): Promise<number> {
const { usableSeconds } = await restore(instanceId, label, 180, log);
return usableSeconds;
}
/**
* keep snapshots an instance as the state to come back to, and records what it was built from.
*
* Called once the caller has brought the scenario to whatever "ready to work" means for it.
*/
export async function keep(
scenario: string,
instanceId: string,
env: NodeJS.ProcessEnv = process.env,
): Promise<Warm> {
await snapshot(instanceId, label);
const warm: Warm = {
scenario,
instanceId,
images: stockOf(instanceId),
against: whatWasTested(env),
at: new Date().toISOString(),
};
remember(warm);
return warm;
}
/** cool destroys what is being kept and forgets it. */
export async function cool(): Promise<string | null> {
const warm = remembered();
forget();
if (!warm) return null;
if ((await standingNow()).includes(warm.instanceId)) {
await destroy(warm.instanceId);
}
return warm.instanceId;
}
/**
* What a raised scenario stocked, held until it is kept.
*
* Raising works the images out and snapshotting happens later, so this carries them between the
* two without the caller having to hold them.
*/
const stock = new Map<string, string[]>();
export function rememberStock(instanceId: string, images: string[]): void {
stock.set(instanceId, images);
}
function stockOf(instanceId: string): string[] {
return stock.get(instanceId) ?? [];
}
/** What a restored instance's registry serves, from when it was warmed. */
export function warmStock(instanceId: string): { images: string[] } {
const warm = remembered();
if (!warm || warm.instanceId !== instanceId) return { images: [] };
return { images: warm.images };
}
+180 -8
View File
@@ -26,6 +26,10 @@ import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
/** Whether this run keeps its mesh for the next one. Off unless asked for. */
const warming = process.env["MESH_LAB_WARM"] === "1";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -128,6 +132,44 @@ function tokenFrom(said: string): string {
before(async () => {
if (skip) return;
// A mesh kept between runs, when one is being kept and still counts.
//
// **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the
// ones after it. Off by default: a run that is meant to mean something raises from nothing,
// because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped
// last week".
if (warming) {
const said = await ready(SCENARIO);
if (said.use === "restore") {
instanceId = said.instanceId;
const seconds = await returnTo(instanceId);
stocked = warmStock(instanceId).images;
// **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
// this suite started by hand is gone — the host most of all. Without it the mesh looks
// perfectly healthy from the control plane's side: a module is assigned, a declaration is
// sent and recorded, and nothing on the machine is listening to apply it. That is exactly
// how this was first met, and it cost an hour to see.
//
// The real answer is a host started by init, which is what the design says it is anyway
// (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places
// it that way, the warm path restarts what it knows it started.
for (const machine of ["anchor", "laptop"]) {
await must(machine, `pgrep -x mesh-host >/dev/null || ` +
`(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
}
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
assert.equal(running.out.trim(), "yes",
"the host did not come back after a restore, so nothing would apply anything");
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
`and started the host again`);
return;
}
console.log(`warm: raising fresh — ${said.why}`);
}
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
instanceId = raised.instanceId;
@@ -153,9 +195,20 @@ before(async () => {
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
}
if (warming) {
// Snapshotted only now, with everything up: a state worth returning to is the one after the
// part nobody wants to repeat.
await rememberStock(instanceId, stocked);
const warm = await keep(SCENARIO, instanceId);
console.log(`warm: ${warm.instanceId} kept, against ` +
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
}
}, { timeout: 1_800_000 });
after(async () => {
// A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything
// else is destroyed, because an instance nobody meant to keep is one nobody will remember.
if (warming) return;
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
@@ -774,16 +827,16 @@ test("rotating a credential moves both ends, and the old one stops working", {
// holding a matching string proves they agree; only an authentication proves they are right.
const store = "/var/lib/mesh/postgres";
await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
`"provides":[{"name":"realpostgres-database","scope":"mesh"}],` +
`"provides":[{"name":"real-postgres-database","scope":"mesh"}],` +
`"capabilities":["container-runtime"],` +
`"serves":{"realpostgres-database":{"port":5433}},` +
`"serves":{"real-postgres-database":{"port":5433}},` +
`"own-secrets":{"superuser":"${store}/superuser"},` +
`"grants":{"realpostgres-database":"${store}/grants"},` +
`"grants":{"real-postgres-database":"${store}/grants"},` +
// Both halves. `grants` is where each consumer's sealed password lands; `receives` is the
// manifest saying who asked and for what. Without the second the provisioner finds a
// directory of unexplained secrets and says nothing has been granted — which is true, and
// reads exactly like a credential that was never delivered.
`"receives":{"realpostgres-database":"${store}/grants/mesh.json"},` +
`"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` +
`"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
`"resources":[` +
`{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
@@ -801,9 +854,9 @@ test("rotating a credential moves both ends, and the old one stops working", {
`"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
`> /tmp/realstore.json`);
await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
`"requires":["realpostgres-database"],"contributes":{"realpostgres-database":{"name":"realapp"}},` +
`"binds":{"realpostgres-database":"/etc/realapp/where.json"},` +
`"secrets":{"realpostgres-database":"/etc/realapp/password"},` +
`"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` +
`"binds":{"real-postgres-database":"/etc/realapp/where.json"},` +
`"secrets":{"real-postgres-database":"/etc/realapp/password"},` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
`> /tmp/realapp.json`);
for (const f of ["realstore", "realapp"]) {
@@ -856,7 +909,7 @@ test("rotating a credential moves both ends, and the old one stops working", {
// Now rotate. One command: the record changes AND both ends are sent, because leaving the
// sending to a later command is the fault above, exactly.
const said = await mesh("rotate realdatabase", 180_000);
const said = await mesh("rotate real-postgres-database", 180_000);
assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
await new Promise((r) => setTimeout(r, 25_000));
@@ -1486,3 +1539,122 @@ test("a service is reached by a name under the machine it runs on", {
}
await mesh("push");
});
// A real third-party workload, adopted the way the conversion will adopt one.
//
// **Everything before this used modules written to exercise the mesh.** This one is software
// nobody here wrote, taking its credentials the way such software does — from its environment —
// and needing two containers that reach each other by name. It is the first module that could not
// have been declared before today: it needs the `network` shape, and it needs a sealed value to
// reach a container's environment.
//
// Its database password is **accepted rather than generated**, which is the whole shape of an
// adoption: a service that already exists keeps the credential it already has, because minting a
// new one is how a running application stops being able to reach its own database.
test("a third-party workload is adopted, with the credential it already had", {
skip, timeout: 900_000,
}, async () => {
const password = "the-password-it-already-had";
await must("anchor", `printf %s ${quote(JSON.stringify({
module: "umami",
version: "1",
capabilities: ["container-runtime"],
"own-secrets": {
database: "/var/lib/umami/database.env",
app: "/var/lib/umami/app.env",
},
listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }],
resources: [
{ id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" },
// The two containers must reach each other by name, which is what this shape is for.
{ id: "net", type: "network", name: "umami" },
{
id: "db", type: "container", name: "umami-db",
image: pinned("postgres"),
network: "umami",
env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" },
"env-file": ["/var/lib/umami/database.env"],
},
{
id: "app", type: "container", name: "umami",
image: pinned("ghcr.io/umami-software/umami"),
network: "umami",
env: { DATABASE_TYPE: "postgresql" },
"env-file": ["/var/lib/umami/app.env"],
ports: ["1212:3000"],
},
],
}))} > /umami.json`);
await must("anchor", `docker cp /umami.json mesh-control:/umami.json`);
await mesh("module add /umami.json");
// **Accepted, not generated.** The value is what the database already answers to; the mesh
// seals it and cannot read it again. Given whole, as the environment lines the containers read.
await must("anchor",
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
`docker exec -i mesh-control /mesh-control secret accept anchor umami database --from -`);
await must("anchor",
`printf %s ${quote(
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
`docker exec -i mesh-control /mesh-control secret accept anchor umami app --from -`);
await mesh("assign anchor umami");
await mesh("push anchor", 300_000);
// Both containers, and the network they share.
let up = false;
for (let i = 0; i < 60 && !up; i++) {
const running = await on("anchor", `docker ps --format '{{.Names}}'`);
up = running.out.includes("umami-db") && running.out.includes("umami");
if (!up) await new Promise((r) => setTimeout(r, 5000));
}
if (!up) {
// Everything that could say why, gathered before asserting. "It did not start" is the one
// thing already known; what is wanted is whether the mesh sent it, whether the host refused
// it, and what the runtime said when it tried.
const said = await mesh("status");
const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`);
const applied = await on("anchor",
`${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`);
const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` +
`for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`);
const tried = await on("anchor",
`docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` +
`docker logs umami-db 2>&1 | tail -15`);
assert.fail(
`the workload never started.\n\n` +
`── what the mesh thinks:\n${said}\n` +
`── containers:\n${containers.out}\n` +
`── what the host owns:\n${applied.out}\n` +
`── what the mesh wrote:\n${files.out}\n` +
`── the database container:\n${tried.out}\n`);
}
// The environment file the mesh sealed is on the machine and readable only by root.
const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`);
assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root");
// **The assertion that matters: the credential works.** Not that a file arrived — that the
// database the mesh started answers to the password the mesh was given rather than one it made.
let connected = { out: "", ok: false };
for (let i = 0; i < 40 && !connected.ok; i++) {
connected = await on("anchor",
`docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`);
if (!connected.ok) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(connected.ok, `the database never came up:\n${connected.out}`);
const wrong = await on("anchor",
`docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` +
`psql -h umami-db -U umami -d umami -qAt -c 'select 1'`);
assert.ok(!wrong.ok,
"the database accepted a password nobody gave it, so this proves nothing about the one that was");
// And the two containers reach each other by name over the module's own network.
const reached = await must("anchor",
`docker run --rm --network umami ${pinned("postgres")} ` +
`sh -c 'getent hosts umami-db || echo unreachable'`);
assert.doesNotMatch(reached, /unreachable/,
"a container could not reach the other by name, so the module's network did nothing");
});
+61
View File
@@ -0,0 +1,61 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { judge, type Warm } from "../src/warm.ts";
const at = "2026-08-31T20:00:00Z";
const built = { "mesh-lab": "aaa", "mesh-host": "bbb", "mesh-control": "ccc" };
const warm = (over: Partial<Warm> = {}): Warm =>
({ scenario: "two-nodes", instanceId: "mlab-two-nodes-1", images: [], against: built, at, ...over });
// The check this exists for: a mesh warmed against code that has since moved would pass today's
// tests against yesterday's binaries, and the result would say nothing about it.
//
// Same fault as novox/hq 04-ISSUES/005, one level down — a green result standing for a run
// against something other than what is in front of you.
test("a warm mesh built from code that has moved is refused", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true,
{ ...built, "mesh-host": "moved" });
assert.equal(said.use, "raise");
assert.match(said.use === "raise" ? said.why : "", /mesh-host was at bbb.*now at moved/);
});
test("a warm mesh built from the same code is used", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, built);
assert.equal(said.use, "restore");
});
// Each refusal is named, because each is a different thing being wrong.
test("every reason to raise instead says which reason it was", () => {
const cases: [string, ReturnType<typeof judge>][] = [
["nothing kept", judge(null, "two-nodes", [], true, built)],
["another scenario", judge(warm({ scenario: "first-node" }), "two-nodes",
["mlab-two-nodes-1"], true, built)],
["not standing", judge(warm(), "two-nodes", [], true, built)],
["no snapshot", judge(warm(), "two-nodes", ["mlab-two-nodes-1"], false, built)],
];
for (const [what, said] of cases) {
assert.equal(said.use, "raise", what);
assert.ok(said.use === "raise" && said.why.length > 10,
`${what} was refused without saying why: ${JSON.stringify(said)}`);
}
});
// A repository the warm record never accounted for is a difference, not a match.
test("a repository that was not recorded when it was warmed is refused", () => {
const said = judge(warm({ against: { "mesh-lab": "aaa" } }), "two-nodes",
["mlab-two-nodes-1"], true, built);
assert.equal(said.use, "raise");
});
// A repository the current run cannot see is not a repository that agrees.
//
// **Found by testing the guard rather than trusting it.** The first version walked only the
// repositories the current environment names, so running without that environment compared
// nothing and called a stale mesh usable. The mesh had genuinely moved; the check had looked at
// neither side.
test("a repository this run cannot locate is refused, not passed over", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, { "mesh-lab": "aaa" });
assert.equal(said.use, "raise");
assert.match(said.use === "raise" ? said.why : "",
/nothing says where it is now|so nothing can say whether it moved/);
});