novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adopt a third-party workload, and keep a mesh between runs

Browse Source 
**The adoption.** Software nobody here wrote, taking its credentials the
way such software does — from its environment — and needing two
containers that reach each other by name. The first module that could
not have been declared this morning: it needs the network shape and it
needs a sealed value to reach a container's environment.

Its password is accepted rather than generated, which is the whole shape
of an adoption: a service that already exists keeps the credential it
already has. Asserted properly — a wrong password is refused by the same
database, so the passing case means something.

**The warm scenario.** A mesh kept between runs and returned to, which
turned twelve minutes of bootstrap into thirty seconds of restore. Off
unless asked for: a run that is meant to mean something raises from
nothing.

Its guard fired for real during this work, unprompted — a mesh-host
commit landed and it refused the stale base, naming both commits, rather
than passing tests against yesterday's binary. That is 04-ISSUES/005's
rule one level down.

Three things the guard learned the hard way and now handles: a snapshot
captures disk and not memory, so the host is restarted after a restore
and asserted to have come back; the stocked image digests are worked out
while raising and a restored instance never raises, so they are kept;
and comparing only the repositories this run can see clears the ones it
cannot, so both directions are compared.

The one real bug behind five failed attempts was in mesh-host and it
reported itself precisely: a network shape the language had and no host
implemented. Everything else was scaffolding of mine.
This commit is contained in:
jschoubben
2026-09-01 01:20:14 +02:00
parent bfcbee49e9
commit a516ee847b
5 changed files with 490 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/integration/mesh.test.ts
+180 -8
View File
@@ -26,6 +26,10 @@ import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
/** Whether this run keeps its mesh for the next one. Off unless asked for. */
const warming = process.env["MESH_LAB_WARM"] === "1";
const capability = await labIsUsable();
const binary = hostBinaryPath();
@@ -128,6 +132,44 @@ function tokenFrom(said: string): string {
before(async () => {
if (skip) return;
// A mesh kept between runs, when one is being kept and still counts.
//
// **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the
// ones after it. Off by default: a run that is meant to mean something raises from nothing,
// because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped
// last week".
if (warming) {
const said = await ready(SCENARIO);
if (said.use === "restore") {
instanceId = said.instanceId;
const seconds = await returnTo(instanceId);
stocked = warmStock(instanceId).images;
// **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
// this suite started by hand is gone — the host most of all. Without it the mesh looks
// perfectly healthy from the control plane's side: a module is assigned, a declaration is
// sent and recorded, and nothing on the machine is listening to apply it. That is exactly
// how this was first met, and it cost an hour to see.
//
// The real answer is a host started by init, which is what the design says it is anyway
// (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places
// it that way, the warm path restarts what it knows it started.
for (const machine of ["anchor", "laptop"]) {
await must(machine, `pgrep -x mesh-host >/dev/null || ` +
`(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
}
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
assert.equal(running.out.trim(), "yes",
"the host did not come back after a restore, so nothing would apply anything");
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
`and started the host again`);
return;
}
console.log(`warm: raising fresh — ${said.why}`);
}
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
instanceId = raised.instanceId;
@@ -153,9 +195,20 @@ before(async () => {
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
}
if (warming) {
// Snapshotted only now, with everything up: a state worth returning to is the one after the
// part nobody wants to repeat.
await rememberStock(instanceId, stocked);
const warm = await keep(SCENARIO, instanceId);
console.log(`warm: ${warm.instanceId} kept, against ` +
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
}
}, { timeout: 1_800_000 });
after(async () => {
// A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything
// else is destroyed, because an instance nobody meant to keep is one nobody will remember.
if (warming) return;
if (instanceId) await destroy(instanceId);
await destroyAll(`${SCENARIO}-`);
}, { timeout: 600_000 });
@@ -774,16 +827,16 @@ test("rotating a credential moves both ends, and the old one stops working", {
// holding a matching string proves they agree; only an authentication proves they are right.
const store = "/var/lib/mesh/postgres";
await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
`"provides":[{"name":"realpostgres-database","scope":"mesh"}],` +
`"provides":[{"name":"real-postgres-database","scope":"mesh"}],` +
`"capabilities":["container-runtime"],` +
`"serves":{"realpostgres-database":{"port":5433}},` +
`"serves":{"real-postgres-database":{"port":5433}},` +
`"own-secrets":{"superuser":"${store}/superuser"},` +
`"grants":{"realpostgres-database":"${store}/grants"},` +
`"grants":{"real-postgres-database":"${store}/grants"},` +
// Both halves. `grants` is where each consumer's sealed password lands; `receives` is the
// manifest saying who asked and for what. Without the second the provisioner finds a
// directory of unexplained secrets and says nothing has been granted — which is true, and
// reads exactly like a credential that was never delivered.
`"receives":{"realpostgres-database":"${store}/grants/mesh.json"},` +
`"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` +
`"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
`"resources":[` +
`{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
@@ -801,9 +854,9 @@ test("rotating a credential moves both ends, and the old one stops working", {
`"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
`> /tmp/realstore.json`);
await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
`"requires":["realpostgres-database"],"contributes":{"realpostgres-database":{"name":"realapp"}},` +
`"binds":{"realpostgres-database":"/etc/realapp/where.json"},` +
`"secrets":{"realpostgres-database":"/etc/realapp/password"},` +
`"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` +
`"binds":{"real-postgres-database":"/etc/realapp/where.json"},` +
`"secrets":{"real-postgres-database":"/etc/realapp/password"},` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
`> /tmp/realapp.json`);
for (const f of ["realstore", "realapp"]) {
@@ -856,7 +909,7 @@ test("rotating a credential moves both ends, and the old one stops working", {
// Now rotate. One command: the record changes AND both ends are sent, because leaving the
// sending to a later command is the fault above, exactly.
const said = await mesh("rotate realdatabase", 180_000);
const said = await mesh("rotate real-postgres-database", 180_000);
assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
await new Promise((r) => setTimeout(r, 25_000));
@@ -1486,3 +1539,122 @@ test("a service is reached by a name under the machine it runs on", {
}
await mesh("push");
});
// A real third-party workload, adopted the way the conversion will adopt one.
//
// **Everything before this used modules written to exercise the mesh.** This one is software
// nobody here wrote, taking its credentials the way such software does — from its environment —
// and needing two containers that reach each other by name. It is the first module that could not
// have been declared before today: it needs the `network` shape, and it needs a sealed value to
// reach a container's environment.
//
// Its database password is **accepted rather than generated**, which is the whole shape of an
// adoption: a service that already exists keeps the credential it already has, because minting a
// new one is how a running application stops being able to reach its own database.
test("a third-party workload is adopted, with the credential it already had", {
skip, timeout: 900_000,
}, async () => {
const password = "the-password-it-already-had";
await must("anchor", `printf %s ${quote(JSON.stringify({
module: "umami",
version: "1",
capabilities: ["container-runtime"],
"own-secrets": {
database: "/var/lib/umami/database.env",
app: "/var/lib/umami/app.env",
},
listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }],
resources: [
{ id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" },
// The two containers must reach each other by name, which is what this shape is for.
{ id: "net", type: "network", name: "umami" },
{
id: "db", type: "container", name: "umami-db",
image: pinned("postgres"),
network: "umami",
env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" },
"env-file": ["/var/lib/umami/database.env"],
},
{
id: "app", type: "container", name: "umami",
image: pinned("ghcr.io/umami-software/umami"),
network: "umami",
env: { DATABASE_TYPE: "postgresql" },
"env-file": ["/var/lib/umami/app.env"],
ports: ["1212:3000"],
},
],
}))} > /umami.json`);
await must("anchor", `docker cp /umami.json mesh-control:/umami.json`);
await mesh("module add /umami.json");
// **Accepted, not generated.** The value is what the database already answers to; the mesh
// seals it and cannot read it again. Given whole, as the environment lines the containers read.
await must("anchor",
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
`docker exec -i mesh-control /mesh-control secret accept anchor umami database --from -`);
await must("anchor",
`printf %s ${quote(
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
`docker exec -i mesh-control /mesh-control secret accept anchor umami app --from -`);
await mesh("assign anchor umami");
await mesh("push anchor", 300_000);
// Both containers, and the network they share.
let up = false;
for (let i = 0; i < 60 && !up; i++) {
const running = await on("anchor", `docker ps --format '{{.Names}}'`);
up = running.out.includes("umami-db") && running.out.includes("umami");
if (!up) await new Promise((r) => setTimeout(r, 5000));
}
if (!up) {
// Everything that could say why, gathered before asserting. "It did not start" is the one
// thing already known; what is wanted is whether the mesh sent it, whether the host refused
// it, and what the runtime said when it tried.
const said = await mesh("status");
const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`);
const applied = await on("anchor",
`${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`);
const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` +
`for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`);
const tried = await on("anchor",
`docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` +
`docker logs umami-db 2>&1 | tail -15`);
assert.fail(
`the workload never started.\n\n` +
`── what the mesh thinks:\n${said}\n` +
`── containers:\n${containers.out}\n` +
`── what the host owns:\n${applied.out}\n` +
`── what the mesh wrote:\n${files.out}\n` +
`── the database container:\n${tried.out}\n`);
}
// The environment file the mesh sealed is on the machine and readable only by root.
const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`);
assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root");
// **The assertion that matters: the credential works.** Not that a file arrived — that the
// database the mesh started answers to the password the mesh was given rather than one it made.
let connected = { out: "", ok: false };
for (let i = 0; i < 40 && !connected.ok; i++) {
connected = await on("anchor",
`docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`);
if (!connected.ok) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(connected.ok, `the database never came up:\n${connected.out}`);
const wrong = await on("anchor",
`docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` +
`psql -h umami-db -U umami -d umami -qAt -c 'select 1'`);
assert.ok(!wrong.ok,
"the database accepted a password nobody gave it, so this proves nothing about the one that was");
// And the two containers reach each other by name over the module's own network.
const reached = await must("anchor",
`docker run --rm --network umami ${pinned("postgres")} ` +
`sh -c 'getent hosts umami-db || echo unreachable'`);
assert.doesNotMatch(reached, /unreachable/,
"a container could not reach the other by name, so the module's network did nothing");
});
test/warm.test.ts
+61
View File
@@ -0,0 +1,61 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { judge, type Warm } from "../src/warm.ts";
const at = "2026-08-31T20:00:00Z";
const built = { "mesh-lab": "aaa", "mesh-host": "bbb", "mesh-control": "ccc" };
const warm = (over: Partial<Warm> = {}): Warm =>
({ scenario: "two-nodes", instanceId: "mlab-two-nodes-1", images: [], against: built, at, ...over });
// The check this exists for: a mesh warmed against code that has since moved would pass today's
// tests against yesterday's binaries, and the result would say nothing about it.
//
// Same fault as novox/hq 04-ISSUES/005, one level down — a green result standing for a run
// against something other than what is in front of you.
test("a warm mesh built from code that has moved is refused", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true,
{ ...built, "mesh-host": "moved" });
assert.equal(said.use, "raise");
assert.match(said.use === "raise" ? said.why : "", /mesh-host was at bbb.*now at moved/);
});
test("a warm mesh built from the same code is used", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, built);
assert.equal(said.use, "restore");
});
// Each refusal is named, because each is a different thing being wrong.
test("every reason to raise instead says which reason it was", () => {
const cases: [string, ReturnType<typeof judge>][] = [
["nothing kept", judge(null, "two-nodes", [], true, built)],
["another scenario", judge(warm({ scenario: "first-node" }), "two-nodes",
["mlab-two-nodes-1"], true, built)],
["not standing", judge(warm(), "two-nodes", [], true, built)],
["no snapshot", judge(warm(), "two-nodes", ["mlab-two-nodes-1"], false, built)],
];
for (const [what, said] of cases) {
assert.equal(said.use, "raise", what);
assert.ok(said.use === "raise" && said.why.length > 10,
`${what} was refused without saying why: ${JSON.stringify(said)}`);
}
});
// A repository the warm record never accounted for is a difference, not a match.
test("a repository that was not recorded when it was warmed is refused", () => {
const said = judge(warm({ against: { "mesh-lab": "aaa" } }), "two-nodes",
["mlab-two-nodes-1"], true, built);
assert.equal(said.use, "raise");
});
// A repository the current run cannot see is not a repository that agrees.
//
// **Found by testing the guard rather than trusting it.** The first version walked only the
// repositories the current environment names, so running without that environment compared
// nothing and called a stale mesh usable. The mesh had genuinely moved; the check had looked at
// neither side.
test("a repository this run cannot locate is refused, not passed over", () => {
const said = judge(warm(), "two-nodes", ["mlab-two-nodes-1"], true, { "mesh-lab": "aaa" });
assert.equal(said.use, "raise");
assert.match(said.use === "raise" ? said.why : "",
/nothing says where it is now|so nothing can say whether it moved/);
});