Compare commits

..
1 Commits
Author SHA1 Message Date
jochen 4ded07bc63 Judge mailu's front by what mail needs, not only its web health
mesh/delivery checking
mesh/merge-gate out of date: judged against trunk ebf7f3773dc7, the trunk is now e846c289c878; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk ebf7f3773dc7, the trunk is now e846c289c878; judged again on a recheck or when it is merged
The image's own check asks nginx's /health and that dovecot runs, so the
front stayed healthy while mail could not flow (issue 688). It now also
needs port 25 to greet over SMTP and the served certificate to hold for
14 more days; certbot renews at 30, so a renewal stuck for two weeks is
said long before the certificate expires.
2026-10-14 02:19:54 +02:00
12 changed files with 29 additions and 759 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
62
66
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
-7
View File
@@ -149,13 +149,6 @@
{
"id": "server",
"type": "container",
"health": {
"kind": "http",
"endpoint": "web",
"path": "/api/healthz",
"status": 200,
"needs": "postgres-database"
},
"name": "gitea",
"image": "gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef",
"env": {
-198
View File
@@ -1,198 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
BASE_IMAGE_ALIAS, BASE_MAX_AGE_DAYS, baseImageFrom, baseProblem, baseScript, buildingBase, cleanCredentials, credentialPath,
describeBase, launchRebuild, launchRun, listBaseBuilds, listRuns, readStatus, runRefusal, running, staleBase,
type BaseImage,
} from "../tools/runs.ts";
// Defends novox/hq issue 698: the lab's base image was 44 days old, a bed machine raised from it carried a
// package database the mirrors no longer served, and genesis-keeps-its-bus stopped on a 404 for
// postgresql-libs. Nothing said how old the image was, nothing refused it, and no mesh tool rebuilt it.
const now = new Date("2026-10-14T01:00:00Z");
const listed = (uploaded: string, alias = BASE_IMAGE_ALIAS) => JSON.stringify([
{ aliases: [{ name: "images:other" }], uploaded_at: "2026-10-13T00:00:00Z" },
{ aliases: [{ name: alias }], uploaded_at: uploaded, created_at: "2026-08-01T00:00:00Z" },
]);
test("the base image's age is read from when this machine made it", () => {
const b = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
assert.equal(b.exists, true);
assert.equal(b.built, "2026-08-31T09:00:00.000Z");
assert.equal(b.ageDays, 43);
assert.match(describeBase(b), /built 2026-08-31T09:00:00.000Z \(43 day\(s\) ago; the bound is 14\)/);
});
test("an image older than the bound refuses a run, naming the tool that rebuilds it", () => {
const stale = baseImageFrom(listed("2026-08-31T09:00:00Z"), now);
const reason = runRefusal(stale);
assert.ok(reason, "a 43-day-old base image was not refused");
assert.match(reason, /older than the lab's bound of 14 days/);
assert.match(reason, /lab_base \{"rebuild": true\}/);
assert.equal(staleBase(stale), reason);
});
test("an image within the bound runs", () => {
const fresh = baseImageFrom(listed(new Date(now.getTime() - BASE_MAX_AGE_DAYS * 86_400_000).toISOString()), now);
assert.equal(fresh.ageDays, BASE_MAX_AGE_DAYS);
assert.equal(runRefusal(fresh), undefined);
});
test("a missing image, or one whose age cannot be read, is said and not refused", () => {
const none = baseImageFrom(listed("2026-10-01T00:00:00Z", "something-else"), now);
assert.equal(none.exists, false);
assert.equal(runRefusal(none), undefined);
assert.match(describeBase(none), /is not on this machine; rebuild it with lab_base/);
const unread = baseImageFrom(undefined, now);
assert.equal(runRefusal(unread), undefined);
assert.match(describeBase(unread), /its age is not known/);
});
test("no run starts while the base image is rebuilt", () => {
const fresh = baseImageFrom(listed("2026-10-13T00:00:00Z"), now);
const reason = runRefusal(fresh, { id: "base-x", started: "2026-10-14T00:59:00Z" });
assert.match(reason ?? "", /is being rebuilt \(base-x/);
});
test("a rebuild is mesh-lab's base build from the forge's main, with no secret in its script", () => {
const s = baseScript("/work/base-x", "https://forge.invalid", "/work/.base-x.git-credentials");
assert.match(s, /clone --quiet --depth 1 'https:\/\/forge.invalid\/novox\/mesh-lab.git'/);
assert.match(s, /src\/cli.ts base build/);
assert.ok(s.indexOf("rm -f '/work/.base-x.git-credentials'") < s.indexOf("npm ci"), "the credential outlives the clone");
});
test("a rebuild is known by its status, and is not a run", () => {
const work = mkdtempSync(join(tmpdir(), "lab-base-"));
try {
mkdirSync(join(work, "base-1"));
writeFileSync(join(work, "base-1", "status.json"), JSON.stringify({ id: "base-1", state: "built", started: "2026-10-13T00:00:00Z" }));
mkdirSync(join(work, "base-2"));
// A build whose process is gone while it says building was lost, and holds nothing up.
writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: 2 ** 22 + 7 }));
assert.deepEqual(listBaseBuilds(work).map((b) => `${b.id} ${b.state}`), ["base-2 lost", "base-1 built"]);
assert.equal(buildingBase(work), undefined);
writeFileSync(join(work, "base-2", "status.json"), JSON.stringify({ id: "base-2", state: "building", started: "2026-10-14T00:00:00Z", pid: process.pid }));
assert.equal(buildingBase(work)?.id, "base-2");
assert.deepEqual(listRuns(work), []);
assert.ok(readFileSync(join(work, "base-1", "status.json"), "utf8"));
} finally {
rmSync(work, { recursive: true, force: true });
}
});
// **A run and a rebuild asked together: one starts** (review 256). Each asks the package registry, which
// waits, and each refused while the other was going; asked at once, both passed their checks before either
// wrote its status. The registry is asked first now, and the check and the start are one step.
function aLab(): string {
return mkdtempSync(join(tmpdir(), "lab-launch-"));
}
const fresh: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-10-13T00:00:00.000Z", ageDays: 1 };
const slowRegistry = () => new Promise<{ registry: string }>((ok) => setTimeout(() => ok({ registry: "https://forge.invalid/r/" }), 20));
const launch = { registry: slowRegistry, base: () => fresh };
function beginRun(work: string, id: string) {
return () => {
mkdirSync(join(work, id));
const s = { id, state: "running" as const, started: new Date().toISOString(), tests: [], refs: {}, pid: process.pid };
writeFileSync(join(work, id, "status.json"), JSON.stringify(s));
return s;
};
}
function beginBuild(work: string, id: string) {
return () => {
mkdirSync(join(work, id));
const b = { id, state: "building" as const, started: new Date().toISOString(), pid: process.pid };
writeFileSync(join(work, id, "status.json"), JSON.stringify(b));
return b;
};
}
test("a run and a rebuild asked at once: exactly one starts", async () => {
for (const runFirst of [true, false]) {
const work = aLab();
try {
// Both asked before either answers, in one order and then the other.
const ask = { run: () => launchRun(work, launch, beginRun(work, "run-a")), build: () => launchRebuild(work, launch, beginBuild(work, "base-a")) };
const [r, b] = runFirst ? await Promise.all([ask.run(), ask.build()])
: (await Promise.all([ask.build(), ask.run()])).reverse() as [Awaited<ReturnType<typeof ask.run>>, Awaited<ReturnType<typeof ask.build>>];
assert.equal([r.started, b.started].filter(Boolean).length, 1, `both or neither started: run ${r.started}, rebuild ${b.started}`);
assert.equal(listRuns(work).length + listBaseBuilds(work).length, 1);
} finally {
rmSync(work, { recursive: true, force: true });
}
}
});
test("two runs, or two rebuilds, asked at once: one of each starts", async () => {
const work = aLab();
try {
const runs = await Promise.all([launchRun(work, launch, beginRun(work, "run-a")), launchRun(work, launch, beginRun(work, "run-b"))]);
assert.deepEqual(runs.map((r) => r.started).sort(), [false, true]);
const other = aLab();
const builds = await Promise.all([launchRebuild(other, launch, beginBuild(other, "base-a")), launchRebuild(other, launch, beginBuild(other, "base-b"))]);
assert.deepEqual(builds.map((b) => b.started).sort(), [false, true]);
rmSync(other, { recursive: true, force: true });
} finally {
rmSync(work, { recursive: true, force: true });
}
});
test("a run is refused on a stale base, and starts nothing", async () => {
const work = aLab();
try {
const stale: BaseImage = { alias: BASE_IMAGE_ALIAS, exists: true, built: "2026-08-31T09:00:00.000Z", ageDays: 43 };
const r = await launchRun(work, { registry: slowRegistry, base: () => stale }, beginRun(work, "run-a"));
assert.equal(r.started, false);
assert.match(r.started ? "" : r.reason, /lab_base \{"rebuild": true\}/);
assert.deepEqual(listRuns(work), []);
} finally {
rmSync(work, { recursive: true, force: true });
}
});
test("a rebuild or a run that never recorded its process is lost, and holds nothing up", () => {
const work = aLab();
try {
mkdirSync(join(work, "base-x"));
writeFileSync(join(work, "base-x", "status.json"), JSON.stringify({ id: "base-x", state: "building", started: "2026-10-14T00:00:00Z" }));
mkdirSync(join(work, "run-x"));
writeFileSync(join(work, "run-x", "status.json"), JSON.stringify({ id: "run-x", state: "running", started: "2026-10-14T00:00:00Z", tests: [], refs: {} }));
assert.equal(listBaseBuilds(work)[0]?.state, "lost");
assert.equal(buildingBase(work), undefined);
assert.equal(readStatus(work, "run-x")?.state, "lost");
assert.equal(running(work), undefined);
} finally {
rmSync(work, { recursive: true, force: true });
}
});
test("a rebuild's credential left behind is cleaned, and a running one's is kept", () => {
const work = aLab();
try {
for (const id of ["base-old", "base-now", "run-old", "run-now"]) writeFileSync(credentialPath(work, id), "x");
const removed = cleanCredentials(work, [credentialPath(work, "base-now"), credentialPath(work, "run-now")]).sort();
assert.deepEqual(removed, [credentialPath(work, "base-old"), credentialPath(work, "run-old")].sort());
} finally {
rmSync(work, { recursive: true, force: true });
}
});
test("lab_check's problem: a stale, missing or unread base image; a fresh one is none", () => {
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false }) ?? "", /is not on this machine/);
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: false, problem: "incus could not list the lab's images" }) ?? "", /age is not known/);
assert.match(baseProblem({ alias: BASE_IMAGE_ALIAS, exists: true, built: "x", ageDays: 30 }) ?? "", /older than the lab's bound/);
assert.equal(baseProblem(fresh), undefined);
});
// The tools use the gate and the check above (index.ts registers with the SDK when imported, so it is read).
test("lab_run and lab_base start through the gate, and lab_check names what is wrong with the base", () => {
const index = readFileSync(new URL("../tools/index.ts", import.meta.url), "utf8");
assert.match(index, /await launchRun\(work,/);
assert.match(index, /await launchRebuild\(work,/);
assert.match(index, /const unfit = baseProblem\(base\);\s*if \(unfit\) problems\.push\(unfit\)/);
assert.match(index, /if \(args\.rebuild !== true\) return reading;/);
});
+1 -106
View File
@@ -5,7 +5,7 @@ import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { installPlan, installStep, NPM_REPOSITORIES, outcomeOf, readStatus, REPOSITORIES, script } from "../tools/runs.ts";
import { installPlan, installStep, NPM_REPOSITORIES, REPOSITORIES, script } from "../tools/runs.ts";
// Defends novox/hq issue 624: lab_run ran `npm ci` at the root of every checkout it installs, and
// mesh-tools keeps its package in node-tools/ with no lock file at its root, so every run failed at its
@@ -381,108 +381,3 @@ test("a forge setting that is not http(s) trusts no origin of its own", async ()
packument("ftp://novox.internal/api/packages/novox/npm/x.tgz")).get, root);
assert.match(viaRoot.problem ?? "", /neither/);
});
// Defends novox/hq issue 693: a run whose only test skipped as NOT REACHED read "0 passed, 0 failed", exit 0,
// and lab_status said passed. The log's tail, as run-2026-10-14T00-21-44-014Z on shanks wrote it.
const oldLog = [
"STOPPED at step 16 of 24, base: \t\t\t\tMake sure you have all the referenced content and try again.",
"\ufe63 genesis adopts its bus as the nats module in place: one bus, its streams kept, the node connected (588152.016874ms) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:",
" \t\t\t\tMake sure you have all the referenced content and try again.",
"NotFound: content digest sha256:e2de96513ba9: not found",
"ℹ tests 1", "ℹ suites 0", "ℹ pass 0", "ℹ fail 0", "ℹ cancelled 0", "ℹ skipped 1", "ℹ todo 0",
"ℹ duration_ms 590254.163993",
"",
"recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, against mesh-lab 7ed7da1",
].join("\n");
// The same run, as the lab's suite says it once it names its skips (mesh-lab, issue 693).
const newLog = [
"ℹ tests 1", "ℹ pass 0", "ℹ fail 0", "ℹ skipped 1",
"",
"recorded: test/integration/genesis-keeps-its-bus.test.ts — 0 passed, 0 failed, 1 skipped, against mesh-lab 7ed7da1",
"NOT PASSED: no test passed: 1 skipped, 1 of them as NOT REACHED",
"skipped: genesis adopts its bus — one bus (test/integration/genesis-keeps-its-bus.test.ts) # ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
].join("\n");
function anEndedRun(log: string, state: "passed" | "failed", exit: number): { work: string; id: string } {
const work = mkdtempSync(join(tmpdir(), "lab-runs-"));
const id = "run-2026-10-14T00-21-44-014Z";
mkdirSync(join(work, id));
writeFileSync(join(work, id, "run.log"), log);
writeFileSync(join(work, id, "status.json"), JSON.stringify({
id, state, exit, started: "2026-10-14T00:21:44.014Z", ended: "2026-10-14T00:33:11.179Z",
tests: ["test/integration/genesis-keeps-its-bus.test.ts"], refs: {},
}));
return { work, id };
}
test("a run whose only test skipped as NOT REACHED, from a lab of before, reads as failed with the reason", () => {
const { work, id } = anEndedRun(oldLog, "passed", 0);
const s = readStatus(work, id);
assert.equal(s?.state, "failed", JSON.stringify(s));
assert.equal(s?.exit, 0);
assert.equal(s?.outcome?.passed, 0);
assert.equal(s?.outcome?.skipped, 1);
assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
assert.match(s?.outcome?.skips[0]?.reason ?? "", /^ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base, before the broker step:$/);
// Kept in the run's record, not only answered.
const kept = JSON.parse(readFileSync(join(work, id, "status.json"), "utf8"));
assert.equal(kept.state, "failed");
assert.equal(kept.outcome.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
rmSync(work, { recursive: true, force: true });
});
test("the lab's own NOT PASSED and skipped lines are read, a test name's punctuation and all", () => {
const { work, id } = anEndedRun(newLog, "failed", 1);
const s = readStatus(work, id);
assert.equal(s?.state, "failed");
assert.equal(s?.outcome?.notPassed, "no test passed: 1 skipped, 1 of them as NOT REACHED");
assert.deepEqual(s?.outcome?.skips, [{
file: "test/integration/genesis-keeps-its-bus.test.ts",
test: "genesis adopts its bus — one bus",
reason: "ADR 0333 §5 NOT REACHED: the first run stopped at step 16 of 24, base: NotFound: content digest",
}]);
rmSync(work, { recursive: true, force: true });
});
test("a run that passed tests and skipped one for a declared reason stays passed, its skip listed", () => {
const log = [
"✔ a module lands (12ms)",
"﹣ a builder builds (0.1ms) # set MESH_LAB_BUILDER to a built mesh-builder",
"ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
].join("\n");
const { work, id } = anEndedRun(log, "passed", 0);
const s = readStatus(work, id);
assert.equal(s?.state, "passed");
assert.equal(s?.outcome?.notPassed, undefined);
assert.equal(s?.outcome?.skips[0]?.reason, "set MESH_LAB_BUILDER to a built mesh-builder");
rmSync(work, { recursive: true, force: true });
});
test("a run that ran no test at all is not passed", () => {
assert.equal(outcomeOf("ℹ tests 0\nℹ pass 0\nℹ fail 0\nℹ skipped 0\n").notPassed, "no test passed: none ran");
});
// Review 254 on issue 693: a test's own output, printed before the suite reports, is not the suite's verdict.
test("a line a test printed that looks like the suite's is not read as the suite's", () => {
const log = [
"NOT PASSED: a test printed this",
"skipped: a forged skip (test/integration/x.test.ts) # FORGED NOT REACHED",
"✔ a module lands (12ms)",
"ℹ tests 1", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 0",
"",
"recorded: test/integration/x.test.ts — 1 passed, 0 failed, 0 skipped, against mesh-lab 7ed7da1",
].join("\n");
const o = outcomeOf(log);
assert.equal(o.notPassed, undefined, JSON.stringify(o));
assert.deepEqual(o.skips, []);
});
test("not reached is matched in any case", () => {
const log = [
"✔ a module lands (12ms)",
"﹣ the broker step (0.1ms) # the broker step was not reached",
"ℹ tests 2", "ℹ pass 1", "ℹ fail 0", "ℹ skipped 1",
].join("\n");
assert.equal(outcomeOf(log).notPassed, "1 test(s) skipped as NOT REACHED: their subject was not reached");
});
+12 -51
View File
@@ -9,8 +9,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import {
checkDir, cleanCredentials, clearChecks, credentialArgs, forgeCredential, installPlan, listRuns, npmRegistry, NPM_REPOSITORIES, publicRootOf, readStatus, REPOSITORIES, running, start,
stop, tail, writeCredential, baseBuildLog, buildingBase, listBaseBuilds, startBaseBuild, credentialPath, launchRun,
launchRebuild, BASE_MAX_AGE_DAYS, describeBase, readBaseImage, baseProblem,
stop, tail, writeCredential,
} from "./runs.js";
export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
@@ -60,18 +59,12 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
// **And what a run would check out and install, before a run finds out** (novox/hq issue 624): every
// repository it clones answers on the forge, and every one it installs has a lock file where the run
// installs from.
// A credential a stopped, crashed or rebooted run or rebuild left behind is removed, and said; one still in
// use by a running run or rebuild is not.
const busy = [running(work)?.id, buildingBase(work)?.id].filter((id): id is string => !!id);
const left = cleanCredentials(work, busy.map((id) => credentialPath(work, id)));
const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run or rebuild left behind: ${left.join(", ")}` : "";
// **And how old the base image is** (novox/hq issue 698): past the bound, missing, or of an age that cannot
// be read is a problem here, though lab_run refuses only the first.
const base = readBaseImage();
const unfit = baseProblem(base);
if (unfit) problems.push(unfit);
const output = [`${check.stdout}${check.stderr}`.trim(), describeBase(base), removed, ...leftovers, ...problems]
.filter(Boolean).join("\n");
// A credential a stopped, crashed or rebooted run left behind is removed, and said; one still in use
// by a running run is not.
const busy = running(work);
const left = cleanCredentials(work, busy ? `${work}/.${busy.id}.git-credentials` : undefined);
const removed = left.length > 0 ? `removed ${left.length} forge credential(s) a run left behind: ${left.join(", ")}` : "";
const output = [`${check.stdout}${check.stderr}`.trim(), removed, ...leftovers, ...problems].filter(Boolean).join("\n");
return { ok: check.status === 0 && problems.length === 0, output };
},
},
@@ -103,43 +96,11 @@ export function getLabTools(env: NodeJS.ProcessEnv): ToolDefinition[] {
}
const stranger = Object.keys(refs).filter((r) => !REPOSITORIES.includes(r));
if (stranger.length > 0) return { started: false, reason: `the lab does not build ${stranger.join(", ")}` };
// **One job at a time, not on a base image past its bound, nor while it is rebuilt** (novox/hq issue 698,
// review 256): the registry asked first, then the check and the start with nothing awaited between.
const launched = await launchRun(work, { registry: () => registryOf(forge), base: () => readBaseImage() },
(registry) => start(work, forge, tests, refs, credentialOf(forge), registry));
if (!launched.started) return launched;
return { started: true, run: launched.job, base: describeBase(launched.base), registry: launched.registry,
...(launched.problem ? { problem: launched.problem } : {}) };
},
},
{
name: "lab_base",
description:
"The lab's base image, which every bed machine with a container runtime is raised from: when it was built, its " +
`age, the bound lab_run refuses past (${BASE_MAX_AGE_DAYS} days), and the last rebuilds with the newest one's log. ` +
"Reads only, unless rebuild is true: then it rebuilds the image with mesh-lab's `base build` from the forge's " +
"main, in the background, and answers at once; ask again for the build's state and the image's new age.",
input: {
rebuild: { type: "boolean", description: "rebuild the image (default false: only read)" },
},
run: async (args) => {
const before = readBaseImage();
const builds = listBaseBuilds(work).slice(0, 3);
const newest = builds[0];
const reading = {
base: before, said: describeBase(before), bound: BASE_MAX_AGE_DAYS,
...(baseProblem(before) ? { problem: baseProblem(before) } : {}),
builds, ...(newest ? { log: baseBuildLog(work, newest.id) } : {}),
};
if (args.rebuild !== true) return reading;
const forge = forgeOf();
if (!forge) return { ...reading, started: false, reason: "the lab's forge is not set: settings for lab, {\"forge\": \"<url>\"}" };
// Not under a run, nor beside another rebuild: the registry asked first, then the check and the start with
// nothing awaited between (review 256).
const launched = await launchRebuild(work, { registry: () => registryOf(forge), base: () => before },
(registry) => startBaseBuild(work, forge, credentialOf(forge), registry));
if (launched.started === false) return { ...reading, started: false, reason: launched.reason };
return { ...reading, started: true, build: launched.job };
const busy = running(work);
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, running: busy };
const registry = await registryOf(forge);
return { started: true, run: start(work, forge, tests, refs, credentialOf(forge), registry.registry),
registry: registry.registry, ...(registry.problem ? { problem: registry.problem } : {}) };
},
},
{
+9 -364
View File
@@ -6,7 +6,7 @@
// and that file is the whole of what the tools read back: a runtime that restarts mid-run still answers
// for it, and says it was lost rather than pretending it is still going.
import { spawn, spawnSync } from "node:child_process";
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { join } from "node:path";
@@ -223,18 +223,16 @@ trap ${shellQuote(`${rm}; exit 129`)} HUP`;
}
/**
* cleanCredentials removes every run's and every base rebuild's credential file left in the work directory
* but the ones named — what a crash or a reboot left behind, which no trap could remove. Answers what it
* removed.
* cleanCredentials removes every run's credential file left in the work directory but the one named — what
* a crash or a reboot left behind, which no trap could remove. Answers what it removed.
*/
export function cleanCredentials(work: string, keep?: string | string[]): string[] {
export function cleanCredentials(work: string, keep?: string): string[] {
if (!existsSync(work)) return [];
const kept = new Set([keep ?? []].flat());
const removed: string[] = [];
for (const name of readdirSync(work)) {
if (!/^\.(run|base)-.*\.git-credentials$/.test(name)) continue;
if (!/^\.run-.*\.git-credentials$/.test(name)) continue;
const path = join(work, name);
if (kept.has(path)) continue;
if (path === keep) continue;
rmSync(path, { force: true });
removed.push(path);
}
@@ -288,117 +286,6 @@ export interface RunStatus {
commits?: Record<string, string>;
exit?: number;
pid?: number;
/** What the suite said of its tests, read from the run's log once it ended (novox/hq issue 693). */
outcome?: Outcome;
}
/** One skipped test, its file and the reason it gave. */
export interface Skip {
file: string;
test: string;
reason: string;
}
/**
* What a run's suite said of its tests: its totals, each skip with its reason, and — when the run did not
* pass although its exit said 0 — why (`notPassed`). Totals the runner did not print are absent.
*/
export interface Outcome {
passed?: number;
failed?: number;
skipped?: number;
skips: Skip[];
notPassed?: string;
}
/**
* outcomeOf reads what a run's log says of its tests (novox/hq issue 693).
*
* **A run in which no test passed has proven nothing**, and the runner exits 0 when every test skipped:
* the genesis bed skipped its only test as "NOT REACHED", the suite printed "0 passed, 0 failed", and the
* lab recorded the run as passed. The lab's suite now says it itself — a "NOT PASSED: <why>" line and a
* "skipped: <test> (<file>) # <reason>" line per skip after its "recorded:" line, and an exit that is not
* 0 — and this reads those lines. For a run of a lab from before (a branch, or an old log), the same rule is
* read from the runner's own output: its summary totals, and its skip lines' first line of reason.
*
* The rule, the lab suite's (mesh-lab src/outcomes.ts): **not passed when no test passed**, or when a test
* skipped as NOT REACHED (its subject was not reached); a skip for another reason beside tests that passed
* leaves the run passed, its reason listed.
*/
export function outcomeOf(log: string): Outcome {
const plain = log.replace(/\u001b\[[0-9;]*m/g, "");
const total = (word: string) => {
// The last summary in the log: the runner's own, once per run.
const all = [...plain.matchAll(new RegExp(`^\\s*(?:\u2139|#)\\s*${word}\\s+(\\d+)\\s*$`, "gm"))];
const last = all[all.length - 1];
return last ? Number(last[1]) : undefined;
};
const outcome: Outcome = { skips: [] };
const passed = total("pass");
const failed = total("fail");
const skipped = total("skipped");
if (passed !== undefined) outcome.passed = passed;
if (failed !== undefined) outcome.failed = failed;
if (skipped !== undefined) outcome.skipped = skipped;
// **The suite's own lines are read only after its last "recorded:" line** (novox/hq issue 693, review
// 254): a test prints what it likes before the suite reports, and a line of its that looked like the
// suite's — "NOT PASSED: …", "skipped: … # NOT REACHED" — would otherwise stand for the suite's verdict.
const recorded = [...plain.matchAll(/^recorded: .*$/gm)];
const lastRecorded = recorded[recorded.length - 1];
const suiteSaid = lastRecorded ? plain.slice((lastRecorded.index ?? 0) + lastRecorded[0].length) : "";
// The suite's own line, "skipped: <test> (<file>) # <reason>": the file in brackets, a .ts, parts it.
for (const m of suiteSaid.matchAll(/^skipped: (.*?) \(([^()]+\.ts)\) # (.*)$/gm)) {
outcome.skips.push({ file: m[2] ?? "", test: m[1] ?? "", reason: m[3] === "no reason given" ? "" : m[3] ?? "" });
}
if (outcome.skips.length === 0) {
// A lab from before names no skip itself; its runner's spec line does: "﹣ <test> (<ms>ms) # <reason>".
for (const m of plain.matchAll(/^\s*\ufe63 (.*?) \([\d.]+m?s\) # (.*)$/gm)) {
const reason = (m[2] ?? "").trim();
outcome.skips.push({ file: "", test: m[1] ?? "", reason: reason === "SKIP" ? "" : reason });
}
}
const said = suiteSaid.match(/^NOT PASSED: (.*)$/m);
if (said) {
outcome.notPassed = (said[1] ?? "").trim();
} else if ((failed ?? 0) === 0) {
// In any case: "not reached" says the same as "NOT REACHED".
const unreached = outcome.skips.filter((s) => s.reason.toUpperCase().includes("NOT REACHED")).length;
if (passed === 0) {
outcome.notPassed = outcome.skips.length > 0 || (skipped ?? 0) > 0
? `no test passed: ${skipped ?? outcome.skips.length} skipped${unreached > 0 ? `, ${unreached} of them as NOT REACHED` : ""}`
: "no test passed: none ran";
} else if (unreached > 0) {
outcome.notPassed = `${unreached} test(s) skipped as NOT REACHED: their subject was not reached`;
}
}
return outcome;
}
/**
* settled is a run that ended, with its outcome read from its log: a run whose exit said 0 and whose suite
* proved nothing reads as failed, with why (novox/hq issue 693). Read once and kept in its status, so the
* record says it too; a run still going, stopped or lost is left as it is.
*/
export function settled(work: string, s: RunStatus): RunStatus {
if (s.outcome || !s.ended || (s.state !== "passed" && s.state !== "failed")) return s;
let log: string;
try {
log = readFileSync(join(runDir(work, s.id), "run.log"), "utf8");
} catch {
return s;
}
const outcome = outcomeOf(log);
const next: RunStatus = { ...s, outcome };
if (next.state === "passed" && outcome.notPassed) next.state = "failed";
try {
writeFileSync(statusPath(work, s.id), JSON.stringify(next, null, 2));
} catch {
// Read again next time: what lab_status answers is the same either way.
}
return next;
}
export function runDir(work: string, id: string): string {
@@ -413,23 +300,16 @@ export function readStatus(work: string, id: string): RunStatus | undefined {
try {
const s = JSON.parse(readFileSync(statusPath(work, id), "utf8")) as RunStatus;
// A run whose process is gone while its status still says it is going was lost — the runtime or
// the machine restarted under it. Said, rather than left reading as running for ever. One that never
// recorded a process (the runtime died between writing its status and starting it) is lost too: it
// would otherwise hold up every run after it (review 256).
if (!["passed", "failed", "stopped", "lost"].includes(s.state) && gone(s.pid)) {
// the machine restarted under it. Said, rather than left reading as running for ever.
if (!["passed", "failed", "stopped", "lost"].includes(s.state) && s.pid && !alive(s.pid)) {
s.state = "lost";
}
return settled(work, s);
return s;
} catch {
return undefined;
}
}
/** gone is whether a job's process is not running: none recorded, or none by that id. */
function gone(pid: number | undefined): boolean {
return !pid || !alive(pid);
}
function alive(pid: number): boolean {
try {
process.kill(pid, 0);
@@ -566,238 +446,3 @@ export function tail(work: string, id: string, lines: number): string {
return "";
}
}
// ---- the base image (novox/hq issue 698)
//
// Every bed machine that needs a container runtime is a clone of one base image, `mesh-lab/base`, which
// mesh-lab's `base build` makes from the distribution's current image. A machine raised from it carries the
// package database the image was built with. On a rolling distribution the mirrors stop serving what an old
// database names within weeks, and a bed that installs a package then fails on a 404 that reads as a mirror
// outage (novox/hq issue 205; seen 2026-10-14 when genesis-keeps-its-bus installed postgresql-libs from a
// database six weeks old). So the lab names the image's age wherever it is asked, refuses a run on an image
// older than the bound, and rebuilds the image when asked to.
//
// **The bound is 14 days** (BASE_MAX_AGE_DAYS). Arch's mirrors keep a package version for as long as it is
// current and drop it soon after it is replaced; an image a fortnight old has reliably installed what beds
// install, and the one that failed was 44 days old. A rebuild takes minutes, so a short bound costs little;
// a run refused is cheaper than a bed that fails an hour in for a reason that is not the change under test.
/** The image every bed machine with a runtime is raised from, as mesh-lab names it (src/lifecycle/place.ts). */
export const BASE_IMAGE_ALIAS = "mesh-lab/base";
/** How old the base image may be, in days, before a run is refused (see the head of this file). */
export const BASE_MAX_AGE_DAYS = 14;
const DAY = 24 * 60 * 60 * 1000;
export interface BaseImage {
alias: string;
/** Whether the image is there at all. */
exists: boolean;
/** When this machine made it, ISO. */
built?: string;
/** Its age in whole days. */
ageDays?: number;
/** Why its age could not be read, when it could not. */
problem?: string;
}
/**
* baseImageFrom reads `incus image list <alias> --format json`: the image whose aliases hold the base's
* name, and when it was made here (its upload, which a publish sets).
*/
export function baseImageFrom(listed: string | undefined, now: Date): BaseImage {
const alias = BASE_IMAGE_ALIAS;
if (listed === undefined) return { alias, exists: false, problem: "incus could not list the lab's images" };
let images: Array<{ aliases?: Array<{ name?: string }>; uploaded_at?: string; created_at?: string }>;
try {
images = JSON.parse(listed);
} catch {
return { alias, exists: false, problem: "incus listed the lab's images as something that is not JSON" };
}
const image = (Array.isArray(images) ? images : []).find((i) => (i.aliases ?? []).some((a) => a.name === alias));
if (!image) return { alias, exists: false };
const made = [image.uploaded_at, image.created_at].map((t) => (t ? Date.parse(t) : NaN))
.find((t) => Number.isFinite(t) && t > 0);
if (made === undefined) return { alias, exists: true, problem: "incus gives no time the image was made" };
return { alias, exists: true, built: new Date(made).toISOString(), ageDays: Math.floor((now.getTime() - made) / DAY) };
}
/** readBaseImage asks incus, on this machine, for the base image. */
export function readBaseImage(now = new Date()): BaseImage {
const listed = spawnSync("incus", ["image", "list", BASE_IMAGE_ALIAS, "--format", "json"], { encoding: "utf8" });
return baseImageFrom(listed.status === 0 ? listed.stdout : undefined, now);
}
/** The base image's age in one sentence, as lab_check and lab_run say it. */
export function describeBase(b: BaseImage): string {
if (b.problem) return `the base image ${b.alias}: its age is not known: ${b.problem}`;
if (!b.exists) return `the base image ${b.alias} is not on this machine; rebuild it with lab_base {"rebuild": true}`;
return `the base image ${b.alias} was built ${b.built} (${b.ageDays} day(s) ago; the bound is ${BASE_MAX_AGE_DAYS})`;
}
/**
* staleBase is why a run is refused for its base image, or undefined when it is not: an image older than the
* bound. An image that is missing or whose age cannot be read is not refused here — a bed that needs it says
* so itself, and a bed that does not need it runs.
*/
export function staleBase(b: BaseImage): string | undefined {
if (!b.exists || b.ageDays === undefined || b.ageDays <= BASE_MAX_AGE_DAYS) return undefined;
return `the base image ${b.alias} was built ${b.built}, ${b.ageDays} days ago, older than the lab's bound of ` +
`${BASE_MAX_AGE_DAYS} days: a bed machine raised from it carries a package database the mirrors no longer serve ` +
`(novox/hq issues 205, 698). Rebuild it with lab_base {"rebuild": true}, then run again`;
}
/**
* baseProblem is what lab_check and lab_base say is wrong with the base image, or undefined: past the bound, not
* on this machine, or of an age that cannot be read (review 256). Only the first refuses a run.
*/
export function baseProblem(b: BaseImage): string | undefined {
if (b.problem || !b.exists) return describeBase(b);
return staleBase(b);
}
/**
* runRefusal is why lab_run starts no run for the base image, or undefined: a rebuild still going, which
* replaces the image a bed raises its machines from, or an image older than the bound.
*/
export function runRefusal(base: BaseImage, rebuilding?: { id: string; started: string }): string | undefined {
if (rebuilding) return `the base image is being rebuilt (${rebuilding.id}, since ${rebuilding.started}); run once lab_base says it is built`;
return staleBase(base);
}
// ---- its rebuild: mesh-lab's `base build`, in the background, as a run is.
export interface BaseBuild {
id: string;
state: "building" | "built" | "failed" | "lost";
started: string;
ended?: string;
exit?: number;
pid?: number;
}
function buildStatusPath(work: string, id: string): string {
return join(work, id, "status.json");
}
/** The base builds, newest first. A build whose process is gone while it says building was lost. */
export function listBaseBuilds(work: string): BaseBuild[] {
if (!existsSync(work)) return [];
const builds: BaseBuild[] = [];
for (const id of readdirSync(work).filter((d) => d.startsWith("base-"))) {
try {
const b = JSON.parse(readFileSync(buildStatusPath(work, id), "utf8")) as BaseBuild;
if (b.state === "building" && gone(b.pid)) b.state = "lost";
builds.push(b);
} catch {
// A directory without a status is not a build.
}
}
return builds.sort((a, b) => b.started.localeCompare(a.started));
}
/** The base build still going, if any. */
export function buildingBase(work: string): BaseBuild | undefined {
return listBaseBuilds(work).find((b) => b.state === "building");
}
/** The last lines of a base build's log. */
export function baseBuildLog(work: string, id: string, lines = 40): string {
try {
return readFileSync(join(work, id, "build.log"), "utf8").split("\n").slice(-Math.max(1, lines)).join("\n");
} catch {
return "";
}
}
/**
* baseScript is what one rebuild runs: mesh-lab's main from the forge, installed the way a run installs it,
* then its `base build`, which replaces the image and reads back that it works before publishing it.
*/
export function baseScript(dir: string, forge: string, credential?: string, registry = `${forge}${NPM_PATH}`): string {
const lab = join(dir, "mesh-lab");
return `set -euo pipefail
export GIT_TERMINAL_PROMPT=0
${credentialGuard(credential)}
cd ${shellQuote(dir)}
git ${credentialArgs(credential)}clone --quiet --depth 1 ${shellQuote(`${forge}/novox/mesh-lab.git`)} ${shellQuote(lab)}
${credential ? `rm -f ${shellQuote(credential)}` : ""}
echo "mesh-lab $(git -C ${shellQuote(lab)} rev-parse HEAD)"
printf '%s\\n' ${shellQuote(`@novox:registry=${registry}`)} > ${shellQuote(join(dir, ".npmrc"))}
export NPM_CONFIG_USERCONFIG=${shellQuote(join(dir, ".npmrc"))}
${installStep(lab, false)}
cd ${shellQuote(lab)}
node --experimental-strip-types src/cli.ts base build
`;
}
/** startBaseBuild rebuilds the base image in the background and answers at once with the build. */
export function startBaseBuild(work: string, forge: string, credential?: string, registry?: string): BaseBuild {
const id = `base-${new Date().toISOString().replace(/[:.]/g, "-")}`;
const dir = join(work, id);
mkdirSync(dir, { recursive: true });
const build: BaseBuild = { id, state: "building", started: new Date().toISOString() };
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
// What a crashed or rebooted run or rebuild left behind goes before this one writes its own.
cleanCredentials(work);
const credentials = writeCredential(credentialPath(work, id), credential);
writeFileSync(join(dir, "build.sh"), baseScript(dir, forge, credentials, registry), { mode: 0o700 });
const wrapper = `bash ${shellQuote(join(dir, "build.sh"))} > ${shellQuote(join(dir, "build.log"))} 2>&1; code=$?
node -e ${shellQuote(
`const f=${JSON.stringify(buildStatusPath(work, id))};const s=JSON.parse(require("fs").readFileSync(f,"utf8"));s.state=process.argv[1]==="0"?"built":"failed";s.exit=Number(process.argv[1]);s.ended=new Date().toISOString();require("fs").writeFileSync(f,JSON.stringify(s,null,2))`,
)} "$code"
rm -f ${shellQuote(credentialPath(work, id))}
rm -rf ${shellQuote(join(dir, "mesh-lab"))}`;
const child = spawn("bash", ["-c", wrapper], { detached: true, stdio: "ignore" });
child.unref();
build.pid = child.pid;
writeFileSync(buildStatusPath(work, id), JSON.stringify(build, null, 2));
return build;
}
// ---- starting a run or a rebuild: one job on the lab's machine at a time (review 256).
//
// **Everything that waits is done first; the check and the start are one step.** A run and a rebuild each
// refused while the other was going, but each asked the package registry — an await — between its check and
// writing its status, so a run and a rebuild asked together could both pass their checks and both start: a
// bed raising machines from an image being replaced under it. Here the registry is asked first, and nothing
// is awaited between reading what is going and writing the new job's status, so in the one runtime that
// serves the lab's tools no other call runs in between.
export interface Launch {
/** The address a run or a rebuild names the package registry by, asked of the forge (npmRegistry). */
registry: () => Promise<{ registry: string; problem?: string }>;
/** The base image as it is now. */
base: () => BaseImage;
}
export type Launched<T> = { started: true; registry: string; problem?: string; base: BaseImage; job: T }
| { started: false; reason: string; base: BaseImage };
/** launchRun starts a run unless a run or a rebuild is going, or the base image is past its bound. */
export async function launchRun(work: string, launch: Launch, begin: (registry: string) => RunStatus):
Promise<Launched<RunStatus>> {
const registry = await launch.registry();
// From here to begin(): nothing awaited.
const base = launch.base();
const busy = running(work);
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}; one run at a time`, base };
const refused = runRefusal(base, buildingBase(work));
if (refused) return { started: false, reason: refused, base };
return { started: true, ...registry, base, job: begin(registry.registry) };
}
/** launchRebuild starts a rebuild of the base image unless a run or another rebuild is going. */
export async function launchRebuild(work: string, launch: Launch, begin: (registry: string) => BaseBuild):
Promise<Launched<BaseBuild>> {
const registry = await launch.registry();
// From here to begin(): nothing awaited.
const base = launch.base();
const rebuilding = buildingBase(work);
if (rebuilding) return { started: false, reason: `${rebuilding.id} is still rebuilding it`, base };
// A run raises its machines from the image; replacing it under one would change what the run tests.
const busy = running(work);
if (busy) return { started: false, reason: `${busy.id} is still ${busy.state}, and raises its machines from this image`, base };
return { started: true, ...registry, base, job: begin(registry.registry) };
}
+6 -1
View File
@@ -560,7 +560,12 @@
"name": "mailu-front",
"image": "ghcr.io/mailu/nginx@sha256:36f98897cd1bc9d27628bbb4e04bdf60147af2ec7507d6da77f002c4f256896d",
"health": {
"kind": "runtime"
"kind": "exec",
"command": "curl -m3 -skfLo /dev/null http://127.0.0.1:10204/health && kill -0 $(cat /run/dovecot/master.pid) && python3 -c \"import smtplib; s = smtplib.SMTP('127.0.0.1', 25, timeout=5); s.ehlo('health'); s.quit()\" && openssl x509 -checkend 1209600 -noout -in /certs/letsencrypt/live/mailu/nginx-chain.pem",
"interval": "60s",
"timeout": "15s",
"looks": 3,
"grace": "120s"
},
"network": "mailu",
"env-file": [
@@ -121,12 +121,6 @@ func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) {
!reflect.DeepEqual(m.Tools, []string{HealthTool, HighestTool}) {
t.Fatalf("watch %v, tools %v", watch, m.Tools)
}
// The watch's verdict is whether the database serves, so its check names the provision it exercises: while the
// database's provider is unhealthy, this finding is held under the provider's condition (hq ADR 0240 rule 5,
// issue 145).
if h["needs"] != "postgres-database" {
t.Fatalf("watch's check needs %v; it exercises postgres-database", h["needs"])
}
if watch["env"].(map[string]any)["MESH_KNOWLEDGE_STATUS"] != m.Build.Artifacts[0].Env["MESH_KNOWLEDGE_STATUS"] {
t.Fatal("watch writes another status file than the health tool reads")
}
-1
View File
@@ -156,7 +156,6 @@
"health": {
"kind": "tool",
"tool": "knowledge_health",
"needs": "postgres-database",
"interval": "60s",
"timeout": "10s",
"looks": 2,
-7
View File
@@ -97,13 +97,6 @@
{
"id": "server",
"type": "container",
"health": {
"kind": "http",
"endpoint": "web",
"path": "/healthz/readiness",
"status": 200,
"needs": "postgres-database"
},
"name": "n8n",
"artifact": "server",
"network": "n8n",
-8
View File
@@ -83,14 +83,6 @@
{
"id": "server",
"type": "container",
"health": {
"kind": "http",
"endpoint": "web",
"path": "/status.php",
"status": 200,
"body": "\"installed\":true",
"needs": "postgres-database"
},
"name": "nextcloud",
"image": "nextcloud@sha256:fb966733647ea03f0446b0c22eac9733c8eb616d37b960caca9d4c3010e14a08",
"env-file": [
-9
View File
@@ -89,15 +89,6 @@
{
"id": "server",
"type": "container",
"health": {
"kind": "http",
"endpoint": "web",
"path": "/api/share/mesh-health",
"status": 404,
"body": "\"code\":\"not-found\"",
"timeout": "10s",
"needs": "postgres-database"
},
"name": "umami",
"image": "ghcr.io/umami-software/umami@sha256:85909afc45bdcda1917394594a087421fdbb05610fded0fa9f6fb861abb2f367",
"network": "umami",