e2e: a running runtime picks up a settings change (issue 009)

Assigns grafana configured by settings, changes the token, pushes again, and asserts the container was replaced (new id) and the rendered config carries the new value. Builds the host from source, since the behaviour under test is the host's. Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
2026-09-04 23:40:13 +02:00
parent 0a414d576a
commit 6067ec1724
1 changed files with 209 additions and 0 deletions
@@ -0,0 +1,209 @@
+/**
+ * A running tool runtime picks up a settings change — novox/hq 04-ISSUES/009, and its fix.
+ *
+ * A runtime reads its settings-merged config file once, at start. Change the settings on an
+ * already-running runtime and, without this, the container keeps the value it read: its spec did
+ * not move (a mounted file's content is not part of it) so the host left it alone, and every check
+ * passed while the mesh did the old thing. The fix gives a container `restart-on`, the same field a
+ * service has: the runtime names its config resource, and the host recreates the container when that
+ * resource changed this pass.
+ *
+ * This assigns grafana configured by settings, then changes the token and pushes again, and asserts
+ * the container was replaced (a new container id) and the config on disk carries the new value.
+ * It builds the host from source (no --no-build), because the behaviour under test is the host's.
+ *
+ *   MESH_LAB_HOST_BINARY=.../mesh-host   MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
+ *   scripts/build-module-runtime.sh grafana builds mesh-runtime-grafana:development, which
+ *   scenarios/grafana-node.yml stocks.
+ */
+
+import { test, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, readFileSync } from "node:fs";
+import { loadScenario } from "../../src/declaration/parse.ts";
+import { raise } from "../../src/lifecycle/raise.ts";
+import { destroy, exec } from "../../src/lifecycle/operate.ts";
+import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
+import { labIsUsable, destroyAll } from "./harness.ts";
+
+const capability = await labIsUsable();
+const binary = hostBinaryPath();
+const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
+
+const skip = !capability.usable
+  ? `lab not usable: ${capability.why}`
+  : !binary || !existsSync(binary)
+    ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
+    : !bundle || !existsSync(bundle)
+      ? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
+      : false;
+
+const SCENARIO = "grafana-node";
+const MACHINE = "anchor";
+
+let instanceId = "";
+let stocked: string[] = [];
+
+function quote(s: string): string {
+  return `'${s.replaceAll("'", `'\\''`)}'`;
+}
+
+async function on(command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
+  const { stdout } = await exec(instanceId, MACHINE, [
+    "sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
+  ], timeoutMs);
+  const marker = stdout.lastIndexOf("__exit=");
+  if (marker < 0) return { out: stdout, ok: false };
+  return { out: stdout.slice(0, marker), ok: stdout.slice(marker + 7).trim() === "0" };
+}
+
+async function must(command: string, timeoutMs?: number): Promise<string> {
+  const { out, ok } = await on(command, timeoutMs);
+  if (!ok) throw new Error(`${MACHINE}: ${command}\n${out}`);
+  return out;
+}
+
+async function mesh(command: string, timeoutMs?: number): Promise<string> {
+  return must(`docker exec mesh-control /mesh-control ${command}`, timeoutMs);
+}
+
+function pinned(repository: string): string {
+  const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
+  assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
+  return found;
+}
+
+function bundleFor(images: string[]): string {
+  let text = readFileSync(bundle, "utf8");
+  for (const ref of images) {
+    const repository = ref.slice(ref.indexOf("/") + 1, ref.indexOf("@"));
+    const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
+    text = text.replaceAll(new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"), ref);
+  }
+  return text;
+}
+
+function tokenFrom(said: string): string {
+  const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
+  assert.ok(found, `no token in:\n${said}`);
+  return found;
+}
+
+async function settled(withinMs = 480_000): Promise<void> {
+  const until = Date.now() + withinMs;
+  let last = "";
+  while (Date.now() < until) {
+    const asked = await on(`docker exec mesh-control /mesh-control status --json`);
+    if (asked.ok) {
+      try {
+        const state = JSON.parse(asked.out) as {
+          wrong: { node: string; outcome: string }[];
+          waiting: { node: string }[];
+          reported: { node: string; outcome: string; current: boolean }[];
+        };
+        const bad = state.wrong.find((w) => w.node === MACHINE);
+        if (bad) throw new Error(`${MACHINE} did not apply what it was sent: ${bad.outcome}\n${asked.out}`);
+        const word = state.reported.find((r) => r.node === MACHINE);
+        if (!state.waiting.some((w) => w.node === MACHINE) && word?.outcome === "applied" && word.current) return;
+        last = asked.out;
+      } catch (err) {
+        if (err instanceof Error && err.message.includes("did not apply")) throw err;
+        last = asked.out;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  throw new Error(`${MACHINE} never caught up within ${Math.round(withinMs / 1000)}s. Last:\n${last}`);
+}
+
+async function setToken(token: string): Promise<void> {
+  const settings = JSON.stringify({ url: "http://127.0.0.1:3000", token });
+  await must(`printf %s ${quote(settings)} > /tmp/s.json && docker cp /tmp/s.json mesh-control:/s.json`);
+  await mesh(`settings set grafana /s.json --node ${MACHINE}`);
+}
+
+async function containerId(): Promise<string> {
+  return (await must(`docker inspect mesh-grafana --format '{{.Id}}'`)).trim();
+}
+
+before(async () => {
+  if (skip) return;
+
+  const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
+    onProgress: (m) => console.log(`raise: ${m}`),
+  });
+  instanceId = raised.instanceId;
+  stocked = raised.images;
+
+  await must(`cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
+  await must(`${HOST_PATH} apply /tmp/substrate.lock`, 600_000);
+  const up = await must(`docker ps --format '{{.Names}}'`);
+  for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
+    assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
+  }
+
+  await mesh(`node add ${MACHINE}`);
+  const token = tokenFrom(await mesh(`token issue --node ${MACHINE}`));
+  await must(`${HOST_PATH} enrol --token ${quote(token)}`);
+  await must(`nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
+}, { timeout: 1_800_000 });
+
+after(async () => {
+  if (instanceId) await destroy(instanceId);
+  await destroyAll(`${SCENARIO}-`);
+}, { timeout: 600_000 });
+
+test("a running runtime is recreated when its settings change, and reads the new value", {
+  skip, timeout: 900_000,
+}, async () => {
+  const manifest = JSON.stringify({
+    module: "grafana",
+    version: "1",
+    emits: ["module.grafana.alert.firing"],
+    "own-secrets": { broker: "/var/lib/mesh/grafana/broker" },
+    resources: [
+      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/grafana", mode: "0700" },
+      { id: "runtime-config", type: "file", path: "/var/lib/mesh/grafana/config.json", mode: "0600", content: "{}\n", merge: "json" },
+      {
+        id: "runtime", type: "container", name: "mesh-grafana", image: pinned("mesh-runtime-grafana"),
+        network: "host",
+        "restart-on": ["runtime-config"],
+        volumes: [
+          "/var/lib/mesh/grafana/broker:/run/secrets/broker:ro",
+          "/var/lib/mesh/grafana/config.json:/run/config/config.json:ro",
+        ],
+        env: { MESH_BROKER_FILE: "/run/secrets/broker", MESH_GRAFANA_CONFIG_FILE: "/run/config/config.json" },
+      },
+    ],
+  });
+  await must(`printf %s ${quote(manifest)} > /tmp/grafana.json && docker cp /tmp/grafana.json mesh-control:/grafana.json`);
+  await mesh("module add /grafana.json");
+
+  await setToken("token-alpha");
+  await mesh(`module issue grafana --node ${MACHINE}`);
+  await mesh(`assign ${MACHINE} grafana`);
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  const before = await containerId();
+  const configBefore = await must(`cat /var/lib/mesh/grafana/config.json`);
+  assert.match(configBefore, /token-alpha/, `first settings not rendered:\n${configBefore}`);
+
+  // Change the setting on the already-running runtime, and push. Nothing about the container's
+  // spec changes — only the content of the file it mounts.
+  await setToken("token-bravo");
+  await mesh(`push ${MACHINE}`);
+  await settled();
+
+  const configAfter = await must(`cat /var/lib/mesh/grafana/config.json`);
+  assert.match(configAfter, /token-bravo/, `the settings change did not re-render the config:\n${configAfter}`);
+
+  const after = await containerId();
+  assert.notEqual(after, before,
+    `the runtime was NOT recreated on a config change (issue 009 not fixed): id stayed ${before}\n` +
+      `host log:\n${(await on(`grep -i grafana /var/log/mesh-host.log | tail -10`)).out}`);
+
+  // And it is running on the new container, so the process re-read the new config.
+  const running = await must(`docker ps --format '{{.Names}}'`);
+  assert.match(running, /mesh-grafana/, "the recreated runtime is not running");
+});