test/integration/mesh.test.ts

@@ -104,6 +104,45 @@ async function mesh(command: string, timeoutMs?: number): Promise<string> {
   return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
 }
 
+/**
+ * Wait until a machine has actually applied what it was last sent.
+ *
+ * **`push` sends; it does not wait.** It prints "sent N resource(s)" and returns, and the machine
+ * applies afterwards. So asserting on what a machine is running immediately after a push is a race
+ * — and the one this fixes lost it silently: `status` still described the *previous* apply, so it
+ * reported nothing wrong while the containers from this declaration did not exist yet.
+ *
+ * Asked of the mesh rather than of the machine, and in its own terms. A node is settled when it is
+ * neither waiting for what it was sent nor wrong about what it applied — the same two questions
+ * `status` answers, read as JSON so a test is not parsing a report meant for a person.
+ *
+ * A machine that reports a failure ends this at once rather than on the timeout: it is not going
+ * to become right by being waited for, and a refusal read after two minutes of polling is the same
+ * refusal, later.
+ */
+async function settled(node: string, withinMs = 240_000): Promise<void> {
+  const until = Date.now() + withinMs;
+  let last = "";
+  while (Date.now() < until) {
+    const said = await mesh("status --json");
+    const state = JSON.parse(said) as {
+      wrong: { node: string; outcome: string; refused?: string;
+               failed?: { id: string; error: string }[] }[];
+      waiting: { node: string; never: boolean }[];
+    };
+    const bad = state.wrong.find((w) => w.node === node);
+    if (bad) {
+      const why = [bad.refused, ...(bad.failed ?? []).map((f) => `${f.id}: ${f.error}`)]
+        .filter(Boolean).join("\n  ");
+      throw new Error(`${node} did not apply what it was sent (${bad.outcome}):\n  ${why}`);
+    }
+    if (!state.waiting.some((w) => w.node === node)) return;
+    last = said;
+    await new Promise((r) => setTimeout(r, 2000));
+  }
+  throw new Error(`${node} never caught up with what it was sent:\n${last}`);
+}
+
 /**
  * The bundle, with every image reference pointed at this scenario's registry.
  *
@@ -1866,13 +1905,19 @@ test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000
   // waited for a database role, so when the containers were never created at all it reported "no
   // login was created" — true, and silent about the reason. A push that was accepted and an apply
   // that worked are different facts, and the second is the one this depends on.
-  const state = await mesh("status");
-  assert.doesNotMatch(state, /failed/i,
-    `the machine did not do what it was told:\n${state}\n\n` +
-      `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
+  //
+  // And waited for, because `push` sends without waiting. Reading `status` the instant it returns
+  // describes the apply *before* this one, which is how this test came to report a missing
+  // container while insisting the machine was fine.
+  await settled("anchor");
   const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out;
+  // Named with what the mesh meant to send, not only with what the machine has. A container that
+  // is absent because the mesh never asked for it and one that is absent because the machine could
+  // not make it are the same sentence here and different faults entirely, and the plan is the only
+  // thing that tells them apart.
   assert.match(running, /\bpostgres\b/,
     `the database module was pushed and no container for it exists:\n${running}\n\n` +
+      `what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` +
       `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
 
   // The database first: until the provisioner has made the login, the forge has nothing to