test/integration/one-node-mesh.test.ts

@@ -38,7 +38,7 @@
  */
 import { test, before, after } from "node:test";
 import assert from "node:assert/strict";
-import { existsSync, writeFileSync, appendFileSync } from "node:fs";
+import { existsSync, readFileSync, writeFileSync, appendFileSync } from "node:fs";
 import { execFileSync } from "node:child_process";
 import { resolve } from "node:path";
 import { loadScenario } from "../../src/declaration/parse.ts";
@@ -134,6 +134,8 @@ const CATALOGUED = "the catalogue holds every module this mesh built";
 const NETWORK = "the machine's networking is what the modules asked for";
 const DECLARED = "every resource the mesh declared is true on the machine";
 const FOLLOWS = "a change to a module's source reaches the machine on its own";
+const STORE_UPGRADES = "the store is upgraded in place, and the controller reads it through the window";
+const BROKER_UPGRADES = "the broker is upgraded in place, and the mesh talks over the window";
 const SURVIVES = "the mesh comes back after the machine reboots";
 const MODULE = { module: "amqp-ping", repo: "mesh-catalog", path: "modules/amqp-ping", container: "amqp-ping" };
 
@@ -413,6 +415,8 @@ const PLAN: { code: string; title: string }[] = [
   { code: "V3", title: NETWORK },
   { code: "V4", title: DECLARED },
   { code: "E1", title: FOLLOWS },
+  { code: "S1", title: STORE_UPGRADES },
+  { code: "S2", title: BROKER_UPGRADES },
   { code: "E2", title: SURVIVES },
 ];
 
@@ -1012,13 +1016,150 @@ before(async () => {
     return `${behind}\n${rolled}\n${after}`;
   });
 
+  // ---- S1. THE STORE IS UPGRADED IN PLACE, THROUGH A STATED WINDOW -----------------------------
+  //
+  // The store the foundation raised is now the `postgres` module (novox/hq issue 051), so it can be
+  // upgraded like any other — and upgrading a SERVER, unlike its provisioner, recreates the
+  // container the control plane keeps its own records in. This is the window: the data survives on
+  // the named volume, and the control plane's connection pool reconnects to the server that comes
+  // back. A benign marker on the server's spec stands in for a version bump; the mechanism — the
+  // applier replacing the container while the volume persists — is the same one a real bump uses.
+  await step("S1", STORE_UPGRADES, FOLLOWS, async () => {
+    const said: string[] = [];
+
+    // 3.4: the foundation is source-tracked now. Before it was adopted, the store was a container
+    // the installer raised with no source the mesh could hold; now it is a module `status` includes
+    // in what can be behind — the question that "could not form" before.
+    const list = await mesh("module list");
+    assert.match(list, /postgres/, `the store is not a module the mesh lists:\n${list}`);
+    assert.match(list, /lavinmq/, `the broker is not a module the mesh lists:\n${list}`);
+    said.push("  source-tracked    postgres and lavinmq are modules the mesh can report behind");
+
+    // What has to survive the window: every database in the store. Counted, not named, so this does
+    // not depend on which consumers happened to ask for one.
+    const count = async () =>
+      (await on(CONTROL,
+        `docker exec mesh-store psql -U postgres -tAc "select count(*) from pg_database where datistemplate=false"`))
+        .out.trim();
+    const before = await count();
+
+    // The mesh's own upgrade path: move the source, build, roll out. Two files move — the
+    // provisioner, so the artifact changes and the build has something to publish (staleness
+    // compares artifacts, not commits, exactly as E1 records), and the server's spec, so the
+    // roll-out recreates mesh-store, which is the window. A benign marker stands in for a version
+    // bump; the applier replacing the container while the volume persists is the same either way.
+    const checkout = resolve(catalogDir, "..");
+    const provisioner = resolve(catalogDir, "postgres", "provisioner", "index.ts");
+    const path = resolve(catalogDir, "postgres", "module.json");
+    appendFileSync(provisioner, `// store upgrade marker ${before}\n`);
+    const bumped = JSON.parse(readFileSync(path, "utf8"));
+    bumped.resources.find((r: { name?: string }) => r.name === "mesh-store").env.MESH_UPGRADE_MARKER = "s1";
+    writeFileSync(path, JSON.stringify(bumped, null, 2));
+    execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" });
+    execFileSync("git", ["-C", checkout, "commit", "-q", "-m",
+      "Upgrade the store, so the mesh rebuilds and recreates it"], { stdio: "pipe" });
+    execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(STORE.repo)], { stdio: "pipe" });
+    const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+
+    await mesh(`module moved postgres ${head}`);
+    const behind = await mesh("status");
+    assert.match(behind, /behind|build --behind/,
+      `the store's source moved and the mesh does not report it behind:\n${behind}`);
+    await mesh(`build --behind --wait 1200s`, 1_500_000);
+    await mesh(`upgrade postgres roll-out`, 900_000);
+    // roll-out rolls out the behind ARTIFACT (the provisioner runtime); the server's change is a
+    // manifest edit, not a new artifact, so a full push is what applies it — recreating mesh-store.
+    await mesh(`push ${CONTROL}`, 600_000);
+    await waitForContainer(CONTROL, "mesh-store");
+
+    // Poll, not a single read: the recreate can settle after waitForContainer sees a container up,
+    // so a single inspect races the replacement. The window is real; this waits for it to close.
+    const deadline = Date.now() + 120_000;
+    let env = "";
+    while (Date.now() < deadline) {
+      env = (await on(CONTROL, `docker inspect mesh-store --format '{{.Config.Env}}'`)).out;
+      if (/MESH_UPGRADE_MARKER=s1/.test(env)) break;
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    assert.match(env, /MESH_UPGRADE_MARKER=s1/,
+      `mesh-store did not come back on the upgraded spec within the window:\n${env}`);
+    // The control plane read through the window: `status` opens all three of its stores, so a clean
+    // answer is proof the pool reconnected to the server that came back.
+    const status = await mesh("status");
+    assert.doesNotMatch(status, /cannot|refused|could not/i,
+      `the control plane did not read through the store window:\n${status}`);
+    const after = await count();
+    assert.equal(after, before,
+      `databases did not survive the store upgrade (before=${before} after=${after})`);
+    said.push("  window            mesh-store recreated, every database kept, the pool reconnected");
+    return said.join("\n");
+  });
+
+  // ---- S2. THE BROKER IS UPGRADED IN PLACE, OVER THE BROKER ------------------------------------
+  //
+  // The harder one: the broker is what the push that upgrades it travels over. Recreating
+  // mesh-broker drops the bus mid-apply, and the machine has to finish the replacement locally and
+  // the mesh reconnect to the broker that comes back. Same benign-marker stand-in for a version
+  // bump; what is under test is that the mesh survives replacing its own bus.
+  await step("S2", BROKER_UPGRADES, STORE_UPGRADES, async () => {
+    const said: string[] = [];
+    // Same upgrade path as S1, for the broker: move the provisioner and the server's spec, build,
+    // roll out. The roll-out recreates mesh-broker, and it is what the roll-out itself travels over,
+    // so this proves the mesh finishes replacing its own bus and reconnects to the one that returns.
+    const checkout = resolve(catalogDir, "..");
+    const provisioner = resolve(catalogDir, "lavinmq", "provisioner", "index.ts");
+    const path = resolve(catalogDir, "lavinmq", "module.json");
+    appendFileSync(provisioner, `// broker upgrade marker\n`);
+    const bumped = JSON.parse(readFileSync(path, "utf8"));
+    bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env = {
+      ...bumped.resources.find((r: { name?: string }) => r.name === "mesh-broker").env,
+      MESH_UPGRADE_MARKER: "s2",
+    };
+    writeFileSync(path, JSON.stringify(bumped, null, 2));
+    execFileSync("git", ["-C", checkout, "add", provisioner, path], { stdio: "pipe" });
+    execFileSync("git", ["-C", checkout, "commit", "-q", "-m",
+      "Upgrade the broker, so the mesh rebuilds and recreates it"], { stdio: "pipe" });
+    execFileSync("git", ["-C", checkout, "push", "-q", "origin", refFor(PROVIDER.repo)], { stdio: "pipe" });
+    const head = execFileSync("git", ["-C", checkout, "rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+
+    await mesh(`module moved lavinmq ${head}`);
+    const behind = await mesh("status");
+    assert.match(behind, /behind|build --behind/,
+      `the broker's source moved and the mesh does not report it behind:\n${behind}`);
+    await mesh(`build --behind --wait 1200s`, 1_500_000);
+    await mesh(`upgrade lavinmq roll-out`, 900_000);
+    // As in S1: the server's manifest change lands on a full push, not the artifact roll-out.
+    await mesh(`push ${CONTROL}`, 600_000);
+    await waitForContainer(CONTROL, "mesh-broker");
+
+    // Poll, not a single read: recreating the broker drops the bus the push travelled over, so the
+    // control plane reconnects and the recreate settles a cycle later than the store's did — the
+    // window here is a reconnection, and it is longer.
+    const deadline = Date.now() + 120_000;
+    let env = "";
+    while (Date.now() < deadline) {
+      env = (await on(CONTROL, `docker inspect mesh-broker --format '{{.Config.Env}}'`)).out;
+      if (/MESH_UPGRADE_MARKER=s2/.test(env)) break;
+      await new Promise((r) => setTimeout(r, 5_000));
+    }
+    assert.match(env, /MESH_UPGRADE_MARKER=s2/,
+      `mesh-broker did not come back on the upgraded spec within the window:\n${env}`);
+    // The mesh talks over the broker that came back: a fresh push composes and delivers, which needs
+    // the bus, so a clean one is proof the reconnection happened.
+    const again = await mesh(`push ${CONTROL}`, 600_000);
+    assert.doesNotMatch(again, /cannot|refused|could not/i,
+      `the mesh did not talk over the broker that came back:\n${again}`);
+    said.push("  window            mesh-broker recreated, the mesh talks over the one that came back");
+    return said.join("\n");
+  });
+
   // ---- 12. AND IT SURVIVES THE MACHINE STOPPING -------------------------------------------------
   //
   // **Never once tested.** A mesh that works until the machine reboots is a demonstration, not
   // something to move real services onto — and the installer is explicit that a host started the
   // way the lab starts it does not survive a reboot, which makes this the check that says whether
   // that matters.
-  await step("E2", SURVIVES, FOLLOWS, async () => {
+  await step("E2", SURVIVES, BROKER_UPGRADES, async () => {
     await restartMachine(CONTROL);
     const missing: string[] = [];
     for (const container of MUST_RUN) {