Feature 57 (config and provisioning): modules declare config and provisions, their files rendered from templates; synced with main (issues of feature 57, 874)

2026-10-16 07:37:43 +00:00
parent 1a2c917abf
commit 9652b2d5ce
771 changed files with 52063 additions and 1774 deletions
@@ -0,0 +1,198 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import {
+  BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath,
+  describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase,
+  type BaseImage,
+} from "../tools/runs.ts";
+
+// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
+// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
+// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it.
+
+const now = new Date("2026-10-14T01:00:00Z");
+const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([
+  { aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" },
+  { aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" },
+]);
+
+test("the base image's age is read from when this machine made it", () => {
+  const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  assert.equal(b.exists, true);
+  assert.equal(b.built, "2026-08-31T09:00:00.000Z");
+  assert.equal(b.ageDays, 43);
+  assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/);
+});
+
+test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => {
+  const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
+  const reason = runRefusal(stale);
+  assert.ok(reason, "a 43-day-old base image was not refused");
+  assert.match(reason, /older than the lab's bound of 14 days/);
+  assert.match(reason, /lab_base \{"rebuild": true\}/);
+  assert.equal(staleBase(stale), reason);
+});
+
+test("an image within the bound runs", () => {
+  const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now);
+  assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS);
+  assert.equal(runRefusal(fresh), undefined);
+});
+
+test("a missing image, or one whose age cannot be read, is said and not refused", () => {
+  const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now);
+  assert.equal(none.exists, false);
+  assert.equal(runRefusal(none), undefined);
+  assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/);
+  const unread = baseImageFrom(undefined, now);
+  assert.equal(runRefusal(unread), undefined);
+  assert.match(describeBase(unread), /its age is not known/);
+});
+
+test("no run starts while the base image is rebuilt", () => {
+  const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now);
+  const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" });
+  assert.match(reason ?? "", /is being rebuilt \(base-x/);
+});
+
+test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
+  const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
+  assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
+  assert.match(s, /src\/cli.ts base build/);
+  assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
+});
+
+test("a rebuild is known by its status, and is not a run", () => {
+  const work = mkdtempSync(join(tmpdir(), "lab-base-"));
+  try {
+    mkdirSync(join(work, "base-1"));
+    writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" }));
+    mkdirSync(join(work, "base-2"));
+    // A build whose process is gone while it says building was lost, and holds nothing up.
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 }));
+    assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]);
+    assert.equal(buildingBase(work), undefined);
+    writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid }));
+    assert.equal(buildingBase(work)?.id, "base-2");
+    assert.deepEqual(listRuns(work), []);
+    assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8"));
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which
+// waits, and each refused while the other was going; asked at once, both passed their checks before either
+// wrote its status. The registry is asked first now, and the check and the start are one step.
+function aLab(): string {
+  return mkdtempSync(join(tmpdir(), "lab-launch-"));
+}
+const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 };
+const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20));
+const launch = { registry: slowRegistry, base: () => fresh };
+function beginRun(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(s));
+    return s;
+  };
+}
+function beginBuild(work: string, id: string) {
+  return () => {
+    mkdirSync(join(work, id));
+    const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid };
+    writeFileSync(join(work, id, "status.json"), JSON.stringify(b));
+    return b;
+  };
+}
+
+test("a run and a rebuild asked at once: exactly one starts", async () => {
+  for (const runFirst of [true, false]) {
+    const work = aLab();
+    try {
+      // Both asked before either answers, in one order and then the other.
+      const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) };
+      const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()])
+        : (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited<ReturnType<typeof ask.run>>, Awaited<ReturnType<typeof ask.build>>];
+      assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`);
+      assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1);
+    } finally {
+      rmSync(work, { recursive: true, force: true });
+    }
+  }
+});
+
+test("two runs, or two rebuilds, asked at once: one of each starts", async () => {
+  const work = aLab();
+  try {
+    const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]);
+    assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]);
+    const other = aLab();
+    const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]);
+    assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]);
+    rmSync(other, { recursive: true, force: true });
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a run is refused on a stale base, and starts nothing", async () => {
+  const work = aLab();
+  try {
+    const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 };
+    const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a"));
+    assert.equal(r.started, false);
+    assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/);
+    assert.deepEqual(listRuns(work), []);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => {
+  const work = aLab();
+  try {
+    mkdirSync(join(work, "base-x"));
+    writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" }));
+    mkdirSync(join(work, "run-x"));
+    writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} }));
+    assert.equal(listBaseBuilds(work)[0]?.state, "lost");
+    assert.equal(buildingBase(work), undefined);
+    assert.equal(readStatus(work, "run-x")?.state, "lost");
+    assert.equal(running(work), undefined);
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => {
+  const work = aLab();
+  try {
+    for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x");
+    const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort();
+    assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort());
+  } finally {
+    rmSync(work, { recursive: true, force: true });
+  }
+});
+
+test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => {
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/);
+  assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/);
+  assert.equal(baseProblem(fresh), undefined);
+});
+
+// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read).
+test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => {
+  const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
+  assert.match(index, /await launchRun\(work,/);
+  assert.match(index, /await launchRebuild\(work,/);
+  assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/);
+  assert.match(index, /if \(args\.rebuild !== true\) return reading;/);
+});
@@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process";
 import { tmpdir } from "node:os";
 import { join } from "node:path";

-import { installPlan, installStep, NPM_REPOSITORIES, REPOSITORIES, script } from "../tools/runs.ts";
+import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts";

 // Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and
 // mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its
@@ -381,3 +381,126 @@ test("a forge setting that is not http(s) trusts no origin of its own", async ()
    packument("ftp://novox.internal/api/packages/novox/npm/x.tgz")).get, root);
  assert.match(viaRoot.problem ?? "", /neither/);
 });
+
+// The runtime's env file keeps its text once, in files/ as its template (novox/hq ADR 0349 §2, issue 656): no
+// file resource carries its text inline, and the build packs files/ into the one archive the controller reads
+// the templates back from.
+test("every file takes its text from a template in files/", () => {
+  const dir = join(import.meta.dirname, "..");
+  const m = JSON.parse(readFileSync(join(dir, "module.json"), "utf8")) as {
+    resources: Record<string, unknown>[]; build: { artifacts: { name: string; kind?: string; from?: string }[] };
+  };
+  for (const r of m.resources) {
+    assert.equal("content" in r, false, `${String(r.id)} keeps its text inline`);
+    if (r.type === "file") assert.ok(existsSync(join(dir, String(r.template))), `${String(r.id)} names ${String(r.template)}`);
+  }
+  const packs = m.build.artifacts.filter((a) => a.kind === "archive" && a.from === "files");
+  assert.deepEqual(packs.map((a) => a.name), ["files"]);
+  const env = m.resources.find((r) => r.id === "runtime-env");
+  assert.match(readFileSync(join(dir, String(env?.template)), "utf8"), /^MESH_LAB_FORGE=\$\{config:forge\}\n/);
+});
+
+// Defends novox/hq issue 693: a run whose only test skipped as NOT REACHED read "0 passed, 0 failed", exit 0,
+// and lab_status said passed. The log's tail, as run-2026-10-14T00-21-44-014Z on shanks wrote it.
+const oldLog = [
+  "STOPPED at step 16 of 24, base:          \t\t\t\tMake sure you have all the referenced content and try again.",
+  "\ufe63 genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected (588152.016874ms) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:",
+  "         \t\t\t\tMake sure you have all the referenced content and try again.",
+  "NotFound: content digest sha256:e2de96513ba9: not found",
+  "ℹ tests 1", "ℹ suites 0", "ℹ pass 0", "ℹ fail 0", "ℹ cancelled 0", "ℹ skipped 1", "ℹ todo 0",
+  "ℹ duration_ms 590254.163993",
+  "",
+  "recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, against mesh-lab 7ed7da1",
+].join("\n");
+
+// The same run, as the lab's suite says it once it names its skips (mesh-lab, issue 693).
+const newLog = [
+  "ℹ tests 1", "ℹ pass 0", "ℹ fail 0", "ℹ skipped 1",
+  "",
+  "recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1",
+  "NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED",
+  "skipped: genesis adopts its bus — one bus (test/integration/genesis-keeps-its-bus.test.ts) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
+].join("\n");
+
+function anEndedRun(log: string, state: "passed" | "failed", exit: number): { work: string; id: string } {
+  const work = mkdtempSync(join(tmpdir(), "lab-runs-"));
+  const id = "run-2026-10-14T00-21-44-014Z";
+  mkdirSync(join(work, id));
+  writeFileSync(join(work, id, "run.log"), log);
+  writeFileSync(join(work, id, "status.json"), JSON.stringify({
+    id, state, exit, started: "2026-10-14T00:21:44.014Z", ended: "2026-10-14T00:33:11.179Z",
+    tests: ["test/integration/genesis-keeps-its-bus.test.ts"], refs: {},
+  }));
+  return { work, id };
+}
+
+test("a run whose only test skipped as NOT REACHED, from a lab of before, reads as failed with the reason", () => {
+  const { work, id } = anEndedRun(oldLog, "passed", 0);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "failed", JSON.stringify(s));
+  assert.equal(s?.exit, 0);
+  assert.equal(s?.outcome?.passed, 0);
+  assert.equal(s?.outcome?.skipped, 1);
+  assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  assert.match(s?.outcome?.skips[0]?.reason ?? "", /^ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:$/);
+  // Kept in the run's record, not only answered.
+  const kept = JSON.parse(readFileSync(join(work, id, "status.json"), "utf8"));
+  assert.equal(kept.state, "failed");
+  assert.equal(kept.outcome.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("the lab's own NOT PASSED and skipped lines are read, a test name's punctuation and all", () => {
+  const { work, id } = anEndedRun(newLog, "failed", 1);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "failed");
+  assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
+  assert.deepEqual(s?.outcome?.skips, [{
+    file: "test/integration/genesis-keeps-its-bus.test.ts",
+    test: "genesis adopts its bus — one bus",
+    reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
+  }]);
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("a run that passed tests and skipped one for a declared reason stays passed, its skip listed", () => {
+  const log = [
+    "✔ a module lands (12ms)",
+    "﹣ a builder builds (0.1ms) # set MESH_LAB_BUILDER to a built mesh-builder",
+    "ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
+  ].join("\n");
+  const { work, id } = anEndedRun(log, "passed", 0);
+  const s = readStatus(work, id);
+  assert.equal(s?.state, "passed");
+  assert.equal(s?.outcome?.notPassed, undefined);
+  assert.equal(s?.outcome?.skips[0]?.reason, "set MESH_LAB_BUILDER to a built mesh-builder");
+  rmSync(work, { recursive: true, force: true });
+});
+
+test("a run that ran no test at all is not passed", () => {
+  assert.equal(outcomeOf("ℹ tests 0\nℹ pass 0\nℹ fail 0\nℹ skipped 0\n").notPassed, "no test passed: none ran");
+});
+
+// Review 254 on issue 693: a test's own output, printed before the suite reports, is not the suite's verdict.
+test("a line a test printed that looks like the suite's is not read as the suite's", () => {
+  const log = [
+    "NOT PASSED: a test printed this",
+    "skipped: a forged skip (test/integration/x.test.ts) # FORGED NOT REACHED",
+    "✔ a module lands (12ms)",
+    "ℹ tests 1", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 0",
+    "",
+    "recorded: test/integration/x.test.ts — 1 passed, 0 failed, 0 skipped, against mesh-lab 7ed7da1",
+  ].join("\n");
+  const o = outcomeOf(log);
+  assert.equal(o.notPassed, undefined, JSON.stringify(o));
+  assert.deepEqual(o.skips, []);
+});
+
+test("not reached is matched in any case", () => {
+  const log = [
+    "✔ a module lands (12ms)",
+    "﹣ the broker step (0.1ms) # the broker step was not reached",
+    "ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
+  ].join("\n");
+  assert.equal(outcomeOf(log).notPassed, "1 test(s) skipped as NOT REACHED: their subject was not reached");
+});