From 440e2653b281a7c27f1087efa7d2430428f0df83 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 16 Sep 2026 21:51:20 +0200 Subject: [PATCH] Phase 3.3/3.4: prove the store and broker upgrade in place, through the window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S1 upgrades the store: a spec change recreates mesh-store (the server the control plane reads from), and asserts the data on the named volume survives and the pool reconnects — the stated window. It also asserts postgres/lavinmq are now source-tracked modules the mesh can report behind (3.4), the question that could not form before adoption. S2 does the same for the broker, the harder case: the push that upgrades it travels over it, so it proves the mesh reconnects to the bus it just replaced. Issue 051 (WBS 3.3, 3.4). Claude-Session: https://claude.ai/code/session_01D6qtiYU3P9jk3pnAXyAFyx --- test/integration/one-node-mesh.test.ts | 145 ++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 2 deletions(-) diff --git a/test/integration/one-node-mesh.test.ts b/test/integration/one-node-mesh.test.ts index 2e0b8a0..6fd0c81 100644 --- a/test/integration/one-node-mesh.test.ts +++ b/test/integration/one-node-mesh.test.ts @@ -38,7 +38,7 @@ */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, writeFileSync, appendFileSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync, appendFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; import { resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; @@ -134,6 +134,8 @@ const CATALOGUED = "the catalogue holds every module this mesh built"; const NETWORK = "the machine's networking is what the modules asked for"; const DECLARED = "every resource the mesh declared is true on the machine"; const FOLLOWS = "a change to a module's source reaches the machine on its own"; +const STORE_UPGRADES = "the store is upgraded in place, and the controller reads it through the window"; +const BROKER_UPGRADES = "the broker is upgraded in place, and the mesh talks over the window"; const SURVIVES = "the mesh comes back after the machine reboots"; const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping", container: "amqp-ping" }; @@ -413,6 +415,8 @@ const PLAN: { code: string; title: string }[] = [ { code: "V3", title: NETWORK }, { code: "V4", title: DECLARED }, { code: "E1", title: FOLLOWS }, + { code: "S1", title: STORE_UPGRADES }, + { code: "S2", title: BROKER_UPGRADES }, { code: "E2", title: SURVIVES }, ]; @@ -1012,13 +1016,150 @@ before(async () => { return `${behind}\n${rolled}\n${after}`; }); + // ---- S1. THE STORE IS UPGRADED IN PLACE, THROUGH A STATED WINDOW ----------------------------- + // + // The store the foundation raised is now the `postgres` module (novox/hq issue 051), so it can be + // upgraded like any other — and upgrading a SERVER, unlike its provisioner, recreates the + // container the control plane keeps its own records in. This is the window: the data survives on + // the named volume, and the control plane's connection pool reconnects to the server that comes + // back. A benign marker on the server's spec stands in for a version bump; the mechanism — the + // applier replacing the container while the volume persists — is the same one a real bump uses. + await step("S1", STORE_UPGRADES, FOLLOWS, async () => { + const said: string[] = []; + + // 3.4: the foundation is source-tracked now. Before it was adopted, the store was a container + // the installer raised with no source the mesh could hold; now it is a module `status` includes + // in what can be behind — the question that "could not form" before. + const list = await mesh("module list"); + assert.match(list, /postgres/, `the store is not a module the mesh lists:\n${list}`); + assert.match(list, /lavinmq/, `the broker is not a module the mesh lists:\n${list}`); + said.push(" source-tracked postgres and lavinmq are modules the mesh can report behind"); + + // What has to survive the window: every database in the store. Counted, not named, so this does + // not depend on which consumers happened to ask for one. + const count = async () => + (await on(CONTROL, + `docker exec mesh-store psql -U postgres -tAc "select count(*) from pg_database where datistemplate=false"`)) + .out.trim(); + const before = await count(); + + // The mesh's own upgrade path: move the source, build, roll out. Two files move — the + // provisioner, so the artifact changes and the build has something to publish (staleness + // compares artifacts, not commits, exactly as E1 records), and the server's spec, so the + // roll-out recreates mesh-store, which is the window. A benign marker stands in for a version + // bump; the applier replacing the container while the volume persists is the same either way. + const checkout = resolve(catalogDir, ".."); + const provisioner = resolve(catalogDir, "postgres", "provisioner", "index.ts"); + const path = resolve(catalogDir, "postgres", "module.json"); + appendFileSync(provisioner, `// store upgrade marker ${before}\n`); + const bumped = JSON.parse(readFileSync(path, "utf8")); + bumped.resources.find((r: { name?: string }) => r.name === "mesh-store").env.MESH_UPGRADE_MARKER = "s1"; + writeFileSync(path, JSON.stringify(bumped, null, 2)); + execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" }); + execFileSync("git", ["-C", checkout, "commit", "-q", "-m", + "Upgrade the store, so the mesh rebuilds and recreates it"], { stdio: "pipe" }); + execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(STORE.repo)], { stdio: "pipe" }); + const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + await mesh(`module moved postgres ${head}`); + const behind = await mesh("status"); + assert.match(behind, /behind|build --behind/, + `the store's source moved and the mesh does not report it behind:\n${behind}`); + await mesh(`build --behind --wait 1200s`, 1_500_000); + await mesh(`upgrade postgres roll-out`, 900_000); + // roll-out rolls out the behind ARTIFACT (the provisioner runtime); the server's change is a + // manifest edit, not a new artifact, so a full push is what applies it — recreating mesh-store. + await mesh(`push ${CONTROL}`, 600_000); + await waitForContainer(CONTROL, "mesh-store"); + + // Poll, not a single read: the recreate can settle after waitForContainer sees a container up, + // so a single inspect races the replacement. The window is real; this waits for it to close. + const deadline = Date.now() + 120_000; + let env = ""; + while (Date.now() < deadline) { + env = (await on(CONTROL, `docker inspect mesh-store --format '{{.Config.Env}}'`)).out; + if (/MESH_UPGRADE_MARKER=s1/.test(env)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.match(env, /MESH_UPGRADE_MARKER=s1/, + `mesh-store did not come back on the upgraded spec within the window:\n${env}`); + // The control plane read through the window: `status` opens all three of its stores, so a clean + // answer is proof the pool reconnected to the server that came back. + const status = await mesh("status"); + assert.doesNotMatch(status, /cannot|refused|could not/i, + `the control plane did not read through the store window:\n${status}`); + const after = await count(); + assert.equal(after, before, + `databases did not survive the store upgrade (before=${before} after=${after})`); + said.push(" window mesh-store recreated, every database kept, the pool reconnected"); + return said.join("\n"); + }); + + // ---- S2. THE BROKER IS UPGRADED IN PLACE, OVER THE BROKER ------------------------------------ + // + // The harder one: the broker is what the push that upgrades it travels over. Recreating + // mesh-broker drops the bus mid-apply, and the machine has to finish the replacement locally and + // the mesh reconnect to the broker that comes back. Same benign-marker stand-in for a version + // bump; what is under test is that the mesh survives replacing its own bus. + await step("S2", BROKER_UPGRADES, STORE_UPGRADES, async () => { + const said: string[] = []; + // Same upgrade path as S1, for the broker: move the provisioner and the server's spec, build, + // roll out. The roll-out recreates mesh-broker, and it is what the roll-out itself travels over, + // so this proves the mesh finishes replacing its own bus and reconnects to the one that returns. + const checkout = resolve(catalogDir, ".."); + const provisioner = resolve(catalogDir, "lavinmq", "provisioner", "index.ts"); + const path = resolve(catalogDir, "lavinmq", "module.json"); + appendFileSync(provisioner, `// broker upgrade marker\n`); + const bumped = JSON.parse(readFileSync(path, "utf8")); + bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env = { + ...bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env, + MESH_UPGRADE_MARKER: "s2", + }; + writeFileSync(path, JSON.stringify(bumped, null, 2)); + execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" }); + execFileSync("git", ["-C", checkout, "commit", "-q", "-m", + "Upgrade the broker, so the mesh rebuilds and recreates it"], { stdio: "pipe" }); + execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(PROVIDER.repo)], { stdio: "pipe" }); + const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + await mesh(`module moved lavinmq ${head}`); + const behind = await mesh("status"); + assert.match(behind, /behind|build --behind/, + `the broker's source moved and the mesh does not report it behind:\n${behind}`); + await mesh(`build --behind --wait 1200s`, 1_500_000); + await mesh(`upgrade lavinmq roll-out`, 900_000); + // As in S1: the server's manifest change lands on a full push, not the artifact roll-out. + await mesh(`push ${CONTROL}`, 600_000); + await waitForContainer(CONTROL, "mesh-broker"); + + // Poll, not a single read: recreating the broker drops the bus the push travelled over, so the + // control plane reconnects and the recreate settles a cycle later than the store's did — the + // window here is a reconnection, and it is longer. + const deadline = Date.now() + 120_000; + let env = ""; + while (Date.now() < deadline) { + env = (await on(CONTROL, `docker inspect mesh-broker --format '{{.Config.Env}}'`)).out; + if (/MESH_UPGRADE_MARKER=s2/.test(env)) break; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.match(env, /MESH_UPGRADE_MARKER=s2/, + `mesh-broker did not come back on the upgraded spec within the window:\n${env}`); + // The mesh talks over the broker that came back: a fresh push composes and delivers, which needs + // the bus, so a clean one is proof the reconnection happened. + const again = await mesh(`push ${CONTROL}`, 600_000); + assert.doesNotMatch(again, /cannot|refused|could not/i, + `the mesh did not talk over the broker that came back:\n${again}`); + said.push(" window mesh-broker recreated, the mesh talks over the one that came back"); + return said.join("\n"); + }); + // ---- 12. AND IT SURVIVES THE MACHINE STOPPING ------------------------------------------------- // // **Never once tested.** A mesh that works until the machine reboots is a demonstration, not // something to move real services onto — and the installer is explicit that a host started the // way the lab starts it does not survive a reboot, which makes this the check that says whether // that matters. - await step("E2", SURVIVES, FOLLOWS, async () => { + await step("E2", SURVIVES, BROKER_UPGRADES, async () => { await restartMachine(CONTROL); const missing: string[] = []; for (const container of MUST_RUN) {