novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Delete the lab's registry, and bootstrap the anchor through the installer #19

Merged
jschoubben merged 14 commits from feat/bed-bootstraps-through-the-installer into main 2026-09-11 19:57:05 +00:00
Conversation 0 Commits 14 Files Changed 96
+161 -1
1 changed files with 161 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a48b60b604 - Show all commits
test/integration/whole-mesh-full.test.ts
+161 -1
View File
@@ -64,6 +64,20 @@ const CONTROL = "novox";
const NODES = ["novox", "ace", "shanks", "g14"];
const HOME_NODES = ["ace", "shanks", "g14"];
/**
* ADR 0056 — the domain each public-facing node composes its routed names under.
*
* **The bed had none, so the ADR was untested by construction.** A module now contributes a `label`
* to `route` and nothing else; the mesh joins it to the node's public domain and the join is the
* whole feature. On a node with no public domain a labelled contribution composes to nothing — no
* host, no route — so every routed module on this bed was silently unreachable and the bed still
* went green. Two nodes face outward here; the workstations do not and get none, which is also part
* of the design being exercised.
*
* `.incus` rather than the real domains: this repository's beds name nothing routable.
*/
const PUBLIC_DOMAIN: Record<string, string> = { novox: "novox.incus", ace: "zurag.incus" };
/** Keep the instance standing and browsable rather than tearing it down. */
const KEEP = !!process.env["MESH_LAB_KEEP"];
const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined);
@@ -95,6 +109,10 @@ const NOVOX: Mod[] = [
{ name: "mongodb", containers: ["mongo", "mesh-mongodb"] },
{ name: "mssql", containers: ["mssql", "mesh-mssql"] },
{ name: "lavinmq", containers: ["lavinmq", "mesh-lavinmq"] },
// ADR 0056: the proxy now REQUIRES an `acme-ca`, so the bed must assign a provider of one or
// route-proxy is unresolvable and takes every routed module down with it. step-ca is that
// provider, on the anchor, at mesh scope.
{ name: "step-ca", containers: ["step-ca"] },
{ name: "route-proxy", containers: ["route-proxy"] },
{ name: "keycloak", containers: ["keycloak", "mesh-keycloak"] },
{ name: "gitea", containers: ["gitea", "mesh-gitea"] },
@@ -127,6 +145,10 @@ const CORE_NOVOX = new Set([
]);
const GAPS_NOVOX = new Set([
"umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder",
// step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in
// mesh-control, and this bed is not the place to discover that a fix has not landed yet. What is
// gated is the half that is decided and cheap — see the ADR 0056 section at the end.
"step-ca",
]);
/** The ace media/home set. */
@@ -329,6 +351,57 @@ async function psMapOf(node: string): Promise<Map<string, string>> {
return map;
}
/**
* ADR 0056: the internal CA is initialised FROM AN OPERATOR'S ROOT — it does not mint its own.
*
* So the bed has to be an operator. The material is made on the anchor with openssl and handed to
* the mesh through the real `secret accept` path, exactly as a person would: the mesh cannot invent
* a PEM, and the random 32 bytes it makes for an own-secret nobody supplied would leave step-ca
* crash-looping on a root key that is not a key.
*/
async function deliverCaRoot(): Promise<boolean> {
const made = await on(CONTROL, [
"set -e",
"mkdir -p /tmp/ca && cd /tmp/ca",
// No trailing newline on a password file: step-ca reads the file as the password itself.
"openssl rand -hex 16 | tr -d '\\n' > key-password",
"openssl ecparam -genkey -name prime256v1 -out root.unenc",
"openssl ec -in root.unenc -aes256 -passout file:key-password -out root.key",
"rm -f root.unenc",
"openssl req -x509 -new -key root.key -passin file:key-password -sha256 -days 3650" +
` -out root.crt -subj "/CN=Mesh Internal CA/O=Novox Mesh Lab"`,
"docker cp /tmp/ca/root.crt mesh-control:/ca-root-cert",
"docker cp /tmp/ca/root.key mesh-control:/ca-root-key",
"docker cp /tmp/ca/key-password mesh-control:/ca-root-key-password",
].join("\n"), 180_000);
if (!made.ok) {
console.log(`CA ROOT NOT MADE on ${CONTROL}:\n${made.out.split("\n").slice(-8).join("\n")}`);
return false;
}
for (const [name, file] of [
["root-cert", "/ca-root-cert"],
["root-key", "/ca-root-key"],
["root-key-password", "/ca-root-key-password"],
] as const) {
try {
await mesh(`secret accept ${CONTROL} step-ca ${name} --from ${file}`);
} catch (err) {
console.log(`CA ROOT ACCEPT FAILED (${name}): ${(err as Error).message.split("\n").slice(0, 3).join(" | ")}`);
return false;
}
}
return true;
}
/** What a module's manifest says its route label is, or "" if it contributes no route. */
function routeLabelOf(name: string): string {
const path = resolve(catalogDir, name, "module.json");
const m = JSON.parse(readFileSync(path, "utf8")) as {
contributes?: { route?: { label?: string } };
};
return m.contributes?.route?.label ?? "";
}
/** A node's overlay (mesh0) address, or "" if it has none yet. */
async function overlayAddr(node: string): Promise<string> {
const out = (await on(node, `ip -4 -o addr show mesh0 2>/dev/null | awk '{print $4}' | cut -d/ -f1`)).out;
@@ -356,7 +429,27 @@ before(async () => {
// fingerprint, not hostname, so only the address needs correcting.
const bundleText = bundleFor(raised.images).replaceAll("192.0.2.10:5671", "192.0.2.20:5671");
await must(CONTROL, `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleText}\nMESHBUNDLE`);
await must(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
// **Belt as well as braces on the registry.** `raise` now refuses to return until every machine
// can fetch a manifest from the scenario registry, so the first attempt should be the only one.
// This retry is here because of what the failure looked like when the guarantee was missing: the
// apply died on a pull, `before` threw, and the instance was left a bare shell — VMs and a
// registry, no substrate, no enrolment, nothing to read. A pull is the one step here that can
// fail for a reason that goes away by itself, so it is the one step worth attempting twice.
{
let applied = false;
let said = "";
for (let attempt = 1; attempt <= 3 && !applied; attempt++) {
const tried = await on(CONTROL, `${HOST_PATH} apply /tmp/substrate.lock`, 900_000);
applied = tried.ok;
said = tried.out;
if (!applied && attempt < 3) {
console.log(`substrate apply attempt ${attempt} failed; retrying in 30s:\n${said.split("\n").slice(-8).join("\n")}`);
await new Promise((r) => setTimeout(r, 30_000));
}
}
assert.ok(applied, `the substrate did not apply on novox after three attempts:\n${said}`);
}
const up = await must(CONTROL, `docker ps --format '{{.Names}}'`);
for (const c of ["mesh-store", "mesh-broker", "mesh-control"]) {
assert.match(up, new RegExp(c), `the substrate did not raise ${c}:\n${up}`);
@@ -367,6 +460,10 @@ before(async () => {
// home→public works. novox enrols too: substrate host and service node at once.
for (const machine of NODES) {
await mesh(`node add ${machine}`);
// ADR 0056: said as soon as the record exists, because everything routed is composed from it.
// A node that faces the outside has one; the workstations do not, and are given none.
const domain = PUBLIC_DOMAIN[machine];
if (domain) await mesh(`node public-domain ${machine} ${domain}`);
const token = tokenFrom(await mesh(`token issue --node ${machine}`));
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`, 180_000);
assert.match(said, new RegExp(`enrolled as ${machine}`), said);
@@ -511,6 +608,10 @@ test("the full mesh forms across the access point and both server sets converge"
}
}
// ADR 0056: the CA's root, before the push that would otherwise deliver a random 32 bytes for it.
const caRootDelivered = assigned["novox"]!.has("step-ca") ? await deliverCaRoot() : false;
if (!caRootDelivered) console.log("ADR 0056: no operator root delivered; step-ca cannot initialise.");
// ONE push per node (workstations first — cheap — then the heavy service nodes).
const pushError: Record<string, string> = {};
for (const node of ["shanks", "g14", "novox", "ace"]) {
@@ -600,6 +701,55 @@ test("the full mesh forms across the access point and both server sets converge"
}
}
// ================================================================================================
// ADR 0056 — ROUTE NAMES AND THE INTERNAL CA. Additive, and deliberately only the cheap half.
//
// What is checked here is the part that is DECIDED and costs one file read: a module contributes a
// LABEL, the node carries a PUBLIC DOMAIN, and the mesh joins them — `<label>.<public-domain>`,
// with `@` composing to the bare domain. That join is what makes a routed module reachable at all,
// and before this bed set a public domain it composed to nothing on every node, silently.
//
// What is NOT checked here is issuance: whether route-proxy actually obtains a certificate from
// step-ca over ACME. That path is being fixed in mesh-control as this is written, and a bed that
// gated on it would be reporting somebody else's in-flight work as this bed's failure.
// ================================================================================================
const adr: string[] = ["================ ADR 0056: LABELLED ROUTES ================"];
const wanted: { node: string; module: string; label: string; name: string }[] = [];
for (const { node, mods } of PLAN) {
const domain = PUBLIC_DOMAIN[node];
if (!domain) continue;
for (const { name } of mods) {
if (!assigned[node]!.has(name)) continue;
const label = routeLabelOf(name);
if (!label) continue;
wanted.push({ node, module: name, label, name: label === "@" ? domain : `${label}.${domain}` });
}
}
// The composed names as the MESH wrote them, read from the proxy's own received-routes file —
// the mesh's answer, on the machine, rather than this test's arithmetic checked against itself.
const routesFile = (await on("novox", `cat /var/lib/route-proxy/routes/mesh.json 2>&1`)).out;
const missing: string[] = [];
const composed: typeof wanted = [];
for (const w of wanted.filter((w) => w.node === "novox")) {
const present = routesFile.includes(`"${w.name}"`);
adr.push(` ${w.module.padEnd(20)} label ${w.label.padEnd(10)} -> ${w.name.padEnd(28)} ${present ? "COMPOSED" : "MISSING"}`);
if (present) composed.push(w);
else missing.push(`${w.module} (${w.label} -> ${w.name})`);
}
// And that the proxy answers for one of them. Over HTTP, on the anchor: a certificate is the
// issuance question, and this one is only whether the name reaches the proxy at all.
const probe = composed[0];
const servedCode = probe
? (await on("novox", `curl -s -o /dev/null -w '%{http_code}' --max-time 10 -H 'Host: ${probe.name}' http://127.0.0.1/`)).out.trim()
: "";
adr.push(`\n proxy answers for ${probe?.name ?? "(nothing composed)"}: ${servedCode || "no answer"}`);
adr.push(` operator root delivered to step-ca: ${caRootDelivered}`);
adr.push(` step-ca container: ${psMaps["novox"]?.get("step-ca") ?? "MISSING"}`);
console.log(adr.join("\n"));
// ================================================================================================
// GATING. The headline gates: the cross-segment overlay must FORM for at least one home node
// (that is the thing this bed exists to prove). Convergence gates on each node's CORE and no
@@ -609,6 +759,16 @@ test("the full mesh forms across the access point and both server sets converge"
assert.ok(anyHomeFormed,
`the overlay did NOT form across the access point — no home node could reach novox over the overlay:\n${overlaySummary}`);
if (!KEEP) {
// ADR 0056, the cheap half. Reported on a KEEP run like everything else there.
assert.deepEqual(missing, [],
`these routed modules composed no name — a label with no public domain to join it to is a ` +
`module nothing can reach, and it fails silently:\n${adr.join("\n")}\n\nroutes file:\n${routesFile}`);
assert.ok(probe && servedCode !== "" && servedCode !== "000",
`the proxy did not answer for ${probe?.name ?? "any composed name"} (got "${servedCode}"). ` +
`The name composes, so this is the proxy, not the join:\n${adr.join("\n")}`);
}
if (!KEEP) {
assert.deepEqual(allProblems, [], `the full mesh did not converge:\n ${allProblems.join("\n ")}\n\n${summary}`);
} else if (allProblems.length) {