novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Phase 3.3/3.4: prove the store and broker upgrade in place, through the window

Browse Source 
S1 upgrades the store: a spec change recreates mesh-store (the server the control
plane reads from), and asserts the data on the named volume survives and the
pool reconnects — the stated window. It also asserts postgres/lavinmq are now
source-tracked modules the mesh can report behind (3.4), the question that could
not form before adoption. S2 does the same for the broker, the harder case: the
push that upgrades it travels over it, so it proves the mesh reconnects to the
bus it just replaced.

Issue 051 (WBS 3.3, 3.4).

Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx
This commit is contained in:
jschoubben
2026-09-16 22:55:56 +02:00
parent 70c1545161
commit 440e2653b2
1 changed files with 143 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/integration/one-node-mesh.test.ts
+143 -2
View File
@@ -38,7 +38,7 @@
*/
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, writeFileSync, appendFileSync } from "node:fs";
import { existsSync, readFileSync, writeFileSync, appendFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { resolve } from "node:path";
import { loadScenario } from "../../src/declaration/parse.ts";
@@ -134,6 +134,8 @@ const CATALOGUED = "the catalogue holds every module this mesh built";
const NETWORK = "the machine's networking is what the modules asked for";
const DECLARED = "every resource the mesh declared is true on the machine";
const FOLLOWS = "a change to a module's source reaches the machine on its own";
const STORE_UPGRADES = "the store is upgraded in place, and the controller reads it through the window";
const BROKER_UPGRADES = "the broker is upgraded in place, and the mesh talks over the window";
const SURVIVES = "the mesh comes back after the machine reboots";
const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping", container: "amqp-ping" };
@@ -413,6 +415,8 @@ const PLAN: { code: string; title: string }[] = [
{ code: "V3", title: NETWORK },
{ code: "V4", title: DECLARED },
{ code: "E1", title: FOLLOWS },
{ code: "S1", title: STORE_UPGRADES },
{ code: "S2", title: BROKER_UPGRADES },
{ code: "E2", title: SURVIVES },
];
@@ -1012,13 +1016,150 @@ before(async () => {
return `${behind}\n${rolled}\n${after}`;
});
// ---- S1. THE STORE IS UPGRADED IN PLACE, THROUGH A STATED WINDOW -----------------------------
//
// The store the foundation raised is now the `postgres` module (novox/hq issue 051), so it can be
// upgraded like any other — and upgrading a SERVER, unlike its provisioner, recreates the
// container the control plane keeps its own records in. This is the window: the data survives on
// the named volume, and the control plane's connection pool reconnects to the server that comes
// back. A benign marker on the server's spec stands in for a version bump; the mechanism — the
// applier replacing the container while the volume persists — is the same one a real bump uses.
await step("S1", STORE_UPGRADES, FOLLOWS, async () => {
const said: string[] = [];
// 3.4: the foundation is source-tracked now. Before it was adopted, the store was a container
// the installer raised with no source the mesh could hold; now it is a module `status` includes
// in what can be behind — the question that "could not form" before.
const list = await mesh("module list");
assert.match(list, /postgres/, `the store is not a module the mesh lists:\n${list}`);
assert.match(list, /lavinmq/, `the broker is not a module the mesh lists:\n${list}`);
said.push(" source-tracked postgres and lavinmq are modules the mesh can report behind");
// What has to survive the window: every database in the store. Counted, not named, so this does
// not depend on which consumers happened to ask for one.
const count = async () =>
(await on(CONTROL,
`docker exec mesh-store psql -U postgres -tAc "select count(*) from pg_database where datistemplate=false"`))
.out.trim();
const before = await count();
// The mesh's own upgrade path: move the source, build, roll out. Two files move — the
// provisioner, so the artifact changes and the build has something to publish (staleness
// compares artifacts, not commits, exactly as E1 records), and the server's spec, so the
// roll-out recreates mesh-store, which is the window. A benign marker stands in for a version
// bump; the applier replacing the container while the volume persists is the same either way.
const checkout = resolve(catalogDir, "..");
const provisioner = resolve(catalogDir, "postgres", "provisioner", "index.ts");
const path = resolve(catalogDir, "postgres", "module.json");
appendFileSync(provisioner, `// store upgrade marker ${before}\n`);
const bumped = JSON.parse(readFileSync(path, "utf8"));
bumped.resources.find((r: { name?: string }) => r.name === "mesh-store").env.MESH_UPGRADE_MARKER = "s1";
writeFileSync(path, JSON.stringify(bumped, null, 2));
execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" });
execFileSync("git", ["-C", checkout, "commit", "-q", "-m",
"Upgrade the store, so the mesh rebuilds and recreates it"], { stdio: "pipe" });
execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(STORE.repo)], { stdio: "pipe" });
const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
await mesh(`module moved postgres ${head}`);
const behind = await mesh("status");
assert.match(behind, /behind|build --behind/,
`the store's source moved and the mesh does not report it behind:\n${behind}`);
await mesh(`build --behind --wait 1200s`, 1_500_000);
await mesh(`upgrade postgres roll-out`, 900_000);
// roll-out rolls out the behind ARTIFACT (the provisioner runtime); the server's change is a
// manifest edit, not a new artifact, so a full push is what applies it — recreating mesh-store.
await mesh(`push ${CONTROL}`, 600_000);
await waitForContainer(CONTROL, "mesh-store");
// Poll, not a single read: the recreate can settle after waitForContainer sees a container up,
// so a single inspect races the replacement. The window is real; this waits for it to close.
const deadline = Date.now() + 120_000;
let env = "";
while (Date.now() < deadline) {
env = (await on(CONTROL, `docker inspect mesh-store --format '{{.Config.Env}}'`)).out;
if (/MESH_UPGRADE_MARKER=s1/.test(env)) break;
await new Promise((r) => setTimeout(r, 5_000));
}
assert.match(env, /MESH_UPGRADE_MARKER=s1/,
`mesh-store did not come back on the upgraded spec within the window:\n${env}`);
// The control plane read through the window: `status` opens all three of its stores, so a clean
// answer is proof the pool reconnected to the server that came back.
const status = await mesh("status");
assert.doesNotMatch(status, /cannot|refused|could not/i,
`the control plane did not read through the store window:\n${status}`);
const after = await count();
assert.equal(after, before,
`databases did not survive the store upgrade (before=${before} after=${after})`);
said.push(" window mesh-store recreated, every database kept, the pool reconnected");
return said.join("\n");
});
// ---- S2. THE BROKER IS UPGRADED IN PLACE, OVER THE BROKER ------------------------------------
//
// The harder one: the broker is what the push that upgrades it travels over. Recreating
// mesh-broker drops the bus mid-apply, and the machine has to finish the replacement locally and
// the mesh reconnect to the broker that comes back. Same benign-marker stand-in for a version
// bump; what is under test is that the mesh survives replacing its own bus.
await step("S2", BROKER_UPGRADES, STORE_UPGRADES, async () => {
const said: string[] = [];
// Same upgrade path as S1, for the broker: move the provisioner and the server's spec, build,
// roll out. The roll-out recreates mesh-broker, and it is what the roll-out itself travels over,
// so this proves the mesh finishes replacing its own bus and reconnects to the one that returns.
const checkout = resolve(catalogDir, "..");
const provisioner = resolve(catalogDir, "lavinmq", "provisioner", "index.ts");
const path = resolve(catalogDir, "lavinmq", "module.json");
appendFileSync(provisioner, `// broker upgrade marker\n`);
const bumped = JSON.parse(readFileSync(path, "utf8"));
bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env = {
...bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env,
MESH_UPGRADE_MARKER: "s2",
};
writeFileSync(path, JSON.stringify(bumped, null, 2));
execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" });
execFileSync("git", ["-C", checkout, "commit", "-q", "-m",
"Upgrade the broker, so the mesh rebuilds and recreates it"], { stdio: "pipe" });
execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(PROVIDER.repo)], { stdio: "pipe" });
const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
await mesh(`module moved lavinmq ${head}`);
const behind = await mesh("status");
assert.match(behind, /behind|build --behind/,
`the broker's source moved and the mesh does not report it behind:\n${behind}`);
await mesh(`build --behind --wait 1200s`, 1_500_000);
await mesh(`upgrade lavinmq roll-out`, 900_000);
// As in S1: the server's manifest change lands on a full push, not the artifact roll-out.
await mesh(`push ${CONTROL}`, 600_000);
await waitForContainer(CONTROL, "mesh-broker");
// Poll, not a single read: recreating the broker drops the bus the push travelled over, so the
// control plane reconnects and the recreate settles a cycle later than the store's did — the
// window here is a reconnection, and it is longer.
const deadline = Date.now() + 120_000;
let env = "";
while (Date.now() < deadline) {
env = (await on(CONTROL, `docker inspect mesh-broker --format '{{.Config.Env}}'`)).out;
if (/MESH_UPGRADE_MARKER=s2/.test(env)) break;
await new Promise((r) => setTimeout(r, 5_000));
}
assert.match(env, /MESH_UPGRADE_MARKER=s2/,
`mesh-broker did not come back on the upgraded spec within the window:\n${env}`);
// The mesh talks over the broker that came back: a fresh push composes and delivers, which needs
// the bus, so a clean one is proof the reconnection happened.
const again = await mesh(`push ${CONTROL}`, 600_000);
assert.doesNotMatch(again, /cannot|refused|could not/i,
`the mesh did not talk over the broker that came back:\n${again}`);
said.push(" window mesh-broker recreated, the mesh talks over the one that came back");
return said.join("\n");
});
// ---- 12. AND IT SURVIVES THE MACHINE STOPPING -------------------------------------------------
//
// **Never once tested.** A mesh that works until the machine reboots is a demonstration, not
// something to move real services onto — and the installer is explicit that a host started the
// way the lab starts it does not survive a reboot, which makes this the check that says whether
// that matters.
await step("E2", SURVIVES, FOLLOWS, async () => {
await step("E2", SURVIVES, BROKER_UPGRADES, async () => {
await restartMachine(CONTROL);
const missing: string[] = [];
for (const container of MUST_RUN) {