jochen
0fcc560fec
mesh/merge-gate head 0fcc560fecd6 merged into trunk e846c289c878 — pass: builds lab → shanks; no bus step; every machine composes with the change as it did…
mesh/repo-check head 0fcc560fecd6 merged into trunk e846c289c878 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A run and a rebuild each asked the package registry between checking for the other and writing their status, so both could start and a bed could raise machines from an image being replaced. The registry is asked first now. A hard-killed rebuild's credential is cleaned like a run's; a job that never recorded its process is lost rather than blocking every run; and lab_check reports a missing or unreadable base image.
199 lines
10 KiB
TypeScript
199 lines
10 KiB
TypeScript
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
|
|
import {
|
|
BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath,
|
|
describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase,
|
|
type BaseImage,
|
|
} from "../tools/runs.ts";
|
|
|
|
// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
|
|
// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
|
|
// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it.
|
|
|
|
const now = new Date("2026-10-14T01:00:00Z");
|
|
const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([
|
|
{ aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" },
|
|
{ aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" },
|
|
]);
|
|
|
|
test("the base image's age is read from when this machine made it", () => {
|
|
const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
|
|
assert.equal(b.exists, true);
|
|
assert.equal(b.built, "2026-08-31T09:00:00.000Z");
|
|
assert.equal(b.ageDays, 43);
|
|
assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/);
|
|
});
|
|
|
|
test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => {
|
|
const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
|
|
const reason = runRefusal(stale);
|
|
assert.ok(reason, "a 43-day-old base image was not refused");
|
|
assert.match(reason, /older than the lab's bound of 14 days/);
|
|
assert.match(reason, /lab_base \{"rebuild": true\}/);
|
|
assert.equal(staleBase(stale), reason);
|
|
});
|
|
|
|
test("an image within the bound runs", () => {
|
|
const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now);
|
|
assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS);
|
|
assert.equal(runRefusal(fresh), undefined);
|
|
});
|
|
|
|
test("a missing image, or one whose age cannot be read, is said and not refused", () => {
|
|
const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now);
|
|
assert.equal(none.exists, false);
|
|
assert.equal(runRefusal(none), undefined);
|
|
assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/);
|
|
const unread = baseImageFrom(undefined, now);
|
|
assert.equal(runRefusal(unread), undefined);
|
|
assert.match(describeBase(unread), /its age is not known/);
|
|
});
|
|
|
|
test("no run starts while the base image is rebuilt", () => {
|
|
const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now);
|
|
const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" });
|
|
assert.match(reason ?? "", /is being rebuilt \(base-x/);
|
|
});
|
|
|
|
test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
|
|
const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
|
|
assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
|
|
assert.match(s, /src\/cli.ts base build/);
|
|
assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
|
|
});
|
|
|
|
test("a rebuild is known by its status, and is not a run", () => {
|
|
const work = mkdtempSync(join(tmpdir(), "lab-base-"));
|
|
try {
|
|
mkdirSync(join(work, "base-1"));
|
|
writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" }));
|
|
mkdirSync(join(work, "base-2"));
|
|
// A build whose process is gone while it says building was lost, and holds nothing up.
|
|
writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 }));
|
|
assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]);
|
|
assert.equal(buildingBase(work), undefined);
|
|
writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid }));
|
|
assert.equal(buildingBase(work)?.id, "base-2");
|
|
assert.deepEqual(listRuns(work), []);
|
|
assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8"));
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which
|
|
// waits, and each refused while the other was going; asked at once, both passed their checks before either
|
|
// wrote its status. The registry is asked first now, and the check and the start are one step.
|
|
function aLab(): string {
|
|
return mkdtempSync(join(tmpdir(), "lab-launch-"));
|
|
}
|
|
const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 };
|
|
const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20));
|
|
const launch = { registry: slowRegistry, base: () => fresh };
|
|
function beginRun(work: string, id: string) {
|
|
return () => {
|
|
mkdirSync(join(work, id));
|
|
const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid };
|
|
writeFileSync(join(work, id, "status.json"), JSON.stringify(s));
|
|
return s;
|
|
};
|
|
}
|
|
function beginBuild(work: string, id: string) {
|
|
return () => {
|
|
mkdirSync(join(work, id));
|
|
const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid };
|
|
writeFileSync(join(work, id, "status.json"), JSON.stringify(b));
|
|
return b;
|
|
};
|
|
}
|
|
|
|
test("a run and a rebuild asked at once: exactly one starts", async () => {
|
|
for (const runFirst of [true, false]) {
|
|
const work = aLab();
|
|
try {
|
|
// Both asked before either answers, in one order and then the other.
|
|
const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) };
|
|
const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()])
|
|
: (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited<ReturnType<typeof ask.run>>, Awaited<ReturnType<typeof ask.build>>];
|
|
assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`);
|
|
assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1);
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
}
|
|
});
|
|
|
|
test("two runs, or two rebuilds, asked at once: one of each starts", async () => {
|
|
const work = aLab();
|
|
try {
|
|
const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]);
|
|
assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]);
|
|
const other = aLab();
|
|
const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]);
|
|
assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]);
|
|
rmSync(other, { recursive: true, force: true });
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a run is refused on a stale base, and starts nothing", async () => {
|
|
const work = aLab();
|
|
try {
|
|
const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 };
|
|
const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a"));
|
|
assert.equal(r.started, false);
|
|
assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/);
|
|
assert.deepEqual(listRuns(work), []);
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => {
|
|
const work = aLab();
|
|
try {
|
|
mkdirSync(join(work, "base-x"));
|
|
writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" }));
|
|
mkdirSync(join(work, "run-x"));
|
|
writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} }));
|
|
assert.equal(listBaseBuilds(work)[0]?.state, "lost");
|
|
assert.equal(buildingBase(work), undefined);
|
|
assert.equal(readStatus(work, "run-x")?.state, "lost");
|
|
assert.equal(running(work), undefined);
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => {
|
|
const work = aLab();
|
|
try {
|
|
for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x");
|
|
const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort();
|
|
assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort());
|
|
} finally {
|
|
rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => {
|
|
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/);
|
|
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/);
|
|
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/);
|
|
assert.equal(baseProblem(fresh), undefined);
|
|
});
|
|
|
|
// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read).
|
|
test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => {
|
|
const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
|
|
assert.match(index, /await launchRun\(work,/);
|
|
assert.match(index, /await launchRebuild\(work,/);
|
|
assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/);
|
|
assert.match(index, /if \(args\.rebuild !== true\) return reading;/);
|
|
});
|