1665 lines
91 KiB
TypeScript
1665 lines
91 KiB
TypeScript
/**
|
|
* A mesh, raised from nothing, joined by two machines, delivering a credential neither the mesh
|
|
* nor the broker can read.
|
|
*
|
|
* Everything before this proves a part. This proves the parts meet — which is the thing the
|
|
* project keeps saying cannot be checked any other way (novox/hq ADR 0001: every fault of
|
|
* 2026-08-22 was found in production because nothing could be stood up locally).
|
|
*
|
|
* It needs a host binary and the foundation bundle:
|
|
*
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host
|
|
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
|
|
*
|
|
* The bundle's image references are rewritten to the ones this scenario's own registry serves.
|
|
* A digest belongs to whatever registry serves it, so a committed bundle names a registry that is
|
|
* not this one — rewriting is what makes it applicable rather than a placeholder to tidy away.
|
|
*/
|
|
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync } from "node:fs";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import type { HeldImage } from "../../src/pinning.ts";
|
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts";
|
|
import { incus } from "../../src/incus/client.ts";
|
|
import { machineName } from "../../src/lifecycle/names.ts";
|
|
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
|
|
|
|
/** Whether this run keeps its mesh for the next one. Off unless asked for. */
|
|
const warming = process.env["MESH_LAB_WARM"] === "1";
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
|
|
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
|
|
|
|
const skip = !capability.usable
|
|
? `lab not usable: ${capability.why}`
|
|
: !binary || !existsSync(binary)
|
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
: !bundle || !existsSync(bundle)
|
|
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
|
|
// The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074).
|
|
: catalogueIsPresent() || false;
|
|
|
|
const SCENARIO = "two-nodes";
|
|
let instanceId = "";
|
|
/**
|
|
* The MESH's own artifact store, once the registry module is running on the anchor.
|
|
*
|
|
* Not a registry the lab raised — there is no longer any such thing. A build publishes into the
|
|
* store the mesh itself runs, which is the only registry that exists outside this repository.
|
|
*/
|
|
const registry = "127.0.0.1:5000";
|
|
/**
|
|
* The registry module's image, pinned upstream, pulled by the machine over its uplink.
|
|
*
|
|
* The digest mesh-catalog's `registry` module pins, so the store the mesh runs here is the store
|
|
* the mesh runs anywhere.
|
|
*/
|
|
const ARTIFACT_STORE =
|
|
"registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373";
|
|
/** The mesh's own images, as the machines hold them. */
|
|
let held: HeldImage[] = [];
|
|
|
|
/** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */
|
|
function pinned(reference: string): string {
|
|
return onTheMachine(reference, held);
|
|
}
|
|
|
|
function quote(s: string): string {
|
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
}
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
// Each part on its own line, and stderr redirected once for the whole script.
|
|
//
|
|
// It was `${command} 2>&1; echo ...` on a single line, which quietly broke every command
|
|
// containing a heredoc: the terminator line became `MARKER 2>&1; echo ...`, matched nothing, and
|
|
// the heredoc swallowed the rest of the script — including the echo. `exec 2>&1` needs no
|
|
// trailing text on the command's last line, so a heredoc terminates where it says it does.
|
|
const { stdout } = await exec(instanceId, machine, [
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) {
|
|
// **Never success.** `Number("")` is 0, so a missing marker used to read as exit 0 — a
|
|
// command whose output was swallowed reported that it worked, which is the one answer a test
|
|
// harness must never give.
|
|
return { out: stdout, ok: false };
|
|
}
|
|
const said = stdout.slice(marker + 7).trim();
|
|
return { out: stdout.slice(0, marker), ok: said === "0" };
|
|
}
|
|
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
|
|
/** The control plane, which runs in a container on the first node. */
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs);
|
|
}
|
|
|
|
/**
|
|
* Wait until a machine has actually applied what it was last sent.
|
|
*
|
|
* **`push` sends; it does not wait.** It prints "sent N resource(s)" and returns, and the machine
|
|
* applies afterwards. So asserting on what a machine is running immediately after a push is a race
|
|
* — and the one this fixes lost it silently: `status` still described the *previous* apply, so it
|
|
* reported nothing wrong while the containers from this declaration did not exist yet.
|
|
*
|
|
* Asked of the mesh rather than of the machine, and in its own terms. A node is settled when it is
|
|
* neither waiting for what it was sent nor wrong about what it applied — the same two questions
|
|
* `status` answers, read as JSON so a test is not parsing a report meant for a person.
|
|
*
|
|
* A machine that reports a failure ends this at once rather than on the timeout: it is not going
|
|
* to become right by being waited for, and a refusal read after two minutes of polling is the same
|
|
* refusal, later.
|
|
*/
|
|
async function settled(node: string, withinMs = 480_000): Promise<void> {
|
|
const until = Date.now() + withinMs;
|
|
let last = "";
|
|
while (Date.now() < until) {
|
|
// **Could not ask** and **asked, and the answer was bad** are different facts, and only the
|
|
// second is this machine's fault. The control plane is a container on the node being polled:
|
|
// while it applies a declaration, an exec into it can lose its fifo to containerd, and a poll
|
|
// loop that treats that as a verdict reports the mesh broken because the question missed.
|
|
// And a poll that *threw* — an exec timeout, a lost fifo — is also "could not ask", not a
|
|
// verdict. The distinction failed once as an IncusError surfacing at minute four of a wait
|
|
// whose machine was merely slow.
|
|
let state: {
|
|
wrong: { node: string; outcome: string; refused?: string;
|
|
failed?: { id: string; error: string }[] }[];
|
|
waiting: { node: string; never: boolean }[];
|
|
reported: { node: string; outcome: string; current: boolean }[];
|
|
} | undefined;
|
|
let said = "";
|
|
try {
|
|
const asked = await on("anchor",
|
|
`docker exec mesh-controller /mesh-controller status --json`);
|
|
said = asked.out;
|
|
// Parsed inside the try on purpose: a truncated answer from a struggling machine is the
|
|
// same fact as no answer, and the likeliest moment for one is exactly the machine this
|
|
// poll is watching.
|
|
if (asked.ok) state = JSON.parse(said);
|
|
} catch (err) {
|
|
said = (err as Error).message;
|
|
}
|
|
if (!state) {
|
|
last = said;
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
continue;
|
|
}
|
|
|
|
const bad = state.wrong.find((w) => w.node === node);
|
|
if (bad) {
|
|
const why = [bad.refused, ...(bad.failed ?? []).map((f) => `${f.id}: ${f.error}`)]
|
|
.filter(Boolean).join("\n ");
|
|
throw new Error(`${node} did not apply what it was sent (${bad.outcome}):\n ${why}`);
|
|
}
|
|
// Caught up is an equality, not an ordering. This compared timestamps once — report newer
|
|
// than send — and lost the race it invited: the previous test's closing apply reported
|
|
// after this test's push, newer and still about the old declaration. The report now names
|
|
// the declaration it applied, and the mesh says whether that is the one it last sent.
|
|
const word = state.reported.find((r) => r.node === node);
|
|
const acted = word?.outcome === "applied" && word.current;
|
|
if (!state.waiting.some((w) => w.node === node) && acted) return;
|
|
last = said;
|
|
// Unhurried on purpose: each poll is an exec into a container on the machine that is busy
|
|
// applying, and asking four times a minute rather than thirty is the difference between
|
|
// observing the apply and competing with it.
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
}
|
|
throw new Error(`${node} never caught up with what it was sent within ` +
|
|
`${Math.round(withinMs / 1000)}s. Last answer, or the reason there was none:\n${last}`);
|
|
}
|
|
|
|
/**
|
|
* The bundle, as a machine should receive it.
|
|
*
|
|
* The committed example was written for a target that had a registry the lab raised. Its two
|
|
* third-party images become upstream references the machine pulls itself; mesh-controller, which
|
|
* exists in no registry, becomes the ID this machine was handed.
|
|
*/
|
|
function bundleFor(images: HeldImage[]): string {
|
|
return foundationBundle(bundle, images);
|
|
}
|
|
|
|
/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
/** The builder started by hand on the anchor, against the foundation broker's plain port on
|
|
* loopback — the one that builds until a builder module can (see the retired test's note). */
|
|
async function startBuilder(): Promise<void> {
|
|
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
|
|
// return, so it is pushed whenever the machine has none.
|
|
if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
|
|
await incus([
|
|
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
|
"--mode", "0755",
|
|
], 180_000);
|
|
}
|
|
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
|
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
|
|
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
|
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
|
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
|
|
// A mesh kept between runs, when one is being kept and still counts.
|
|
//
|
|
// **Bootstrapping proves the same thing every time**, and the tests worth iterating on are the
|
|
// ones after it. Off by default: a run that is meant to mean something raises from nothing,
|
|
// because "it passes" must not come to mean "it passes against a mesh somebody bootstrapped
|
|
// last week".
|
|
if (warming) {
|
|
const said = await ready(SCENARIO);
|
|
if (said.use === "restore") {
|
|
instanceId = said.instanceId;
|
|
const seconds = await returnTo(instanceId);
|
|
held = warmStock(instanceId).images;
|
|
|
|
// **A snapshot captures disk, not memory.** Restoring reboots the machine, so everything
|
|
// this suite started by hand is gone — the host most of all. Without it the mesh looks
|
|
// perfectly healthy from the control plane's side: a module is assigned, a declaration is
|
|
// sent and recorded, and nothing on the machine is listening to apply it. That is exactly
|
|
// how this was first met, and it cost an hour to see.
|
|
//
|
|
// The real answer is a host started by init, which is what the design says it is anyway
|
|
// (novox/hq 05-the-node-host: a root service, installed as a package). Until the lab places
|
|
// it that way, the warm path restarts what it knows it started.
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
await must(machine, `pgrep -x mesh-host >/dev/null || ` +
|
|
`(nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
|
}
|
|
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
|
|
assert.equal(running.out.trim(), "yes",
|
|
"the host did not come back after a restore, so nothing would apply anything");
|
|
// The hand-started builder is memory too, and the snapshot is disk.
|
|
if (builder) await startBuilder();
|
|
|
|
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
|
|
`and started the host again`);
|
|
return;
|
|
}
|
|
console.log(`warm: raising fresh — ${said.why}`);
|
|
}
|
|
|
|
// **Progress is printed, and that is not decoration** (novox/hq 04-ISSUES/024). A raise takes
|
|
// minutes and said nothing until it finished, so a stall and ordinary work were the same
|
|
// thing to look at — and the one time it mattered, thirty-five minutes of nothing was read as
|
|
// a slow test until somebody went and looked inside the machine.
|
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {
|
|
onProgress: (m) => console.log(`raise: ${m}`),
|
|
});
|
|
instanceId = raised.instanceId;
|
|
|
|
// The first node raises everything from a file rather than from a bundle built into the binary,
|
|
// because the control plane's image is named by the ID this machine holds it under, which is not
|
|
// knowable until it has been handed over.
|
|
held = raised.images;
|
|
await must("anchor", `cat > /tmp/foundation.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
await must("anchor", `${HOST_PATH} apply /tmp/foundation.lock`);
|
|
|
|
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
|
// assumed: nothing else in this scenario would start one.
|
|
if (builder) await startBuilder();
|
|
if (warming) {
|
|
// Snapshotted only now, with everything up: a state worth returning to is the one after the
|
|
// part nobody wants to repeat.
|
|
await rememberStock(instanceId, held);
|
|
const warm = await keep(SCENARIO, instanceId);
|
|
console.log(`warm: ${warm.instanceId} kept, against ` +
|
|
Object.entries(warm.against).map(([n, c]) => `${n} ${c}`).join(", "));
|
|
}
|
|
}, { timeout: 1_800_000 });
|
|
|
|
after(async () => {
|
|
// A kept instance survives on purpose, and `mesh-lab warm cool` is how it goes away. Everything
|
|
// else is destroyed, because an instance nobody meant to keep is one nobody will remember.
|
|
if (warming) return;
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 600_000 });
|
|
|
|
test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
|
|
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
for (const container of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
assert.match(running, new RegExp(container), `${container} is not running`);
|
|
}
|
|
// Answering, not merely up. A container that is running is not a control plane that replies —
|
|
// a distinction this project has already paid for once.
|
|
assert.ok((await mesh("status")).length > 0);
|
|
});
|
|
|
|
test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
|
|
for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
|
|
await mesh(`node add ${node}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${node}`));
|
|
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
|
// found the first time it was run.
|
|
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
|
assert.match(said, new RegExp(`enrolled as ${node}`), said);
|
|
assert.match(said, /sealing key/, "no sealing key was generated");
|
|
}
|
|
|
|
const recorded = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select name from node where sealing_key is not null order by name"`);
|
|
assert.equal(recorded.trim().split("\n").map((l) => l.trim()).sort().join(","), "anchor,laptop",
|
|
"the mesh did not record a sealing key for both machines");
|
|
});
|
|
|
|
test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
|
|
// The whole argument, on real machines: the two ends must hold the SAME password, and it must
|
|
// appear nowhere the mesh or the broker could read it.
|
|
await must("anchor", `printf %s '{"module":"a-store","version":"1",` +
|
|
`"provides":[{"name":"postgres-database","scope":"mesh"}],"serves":{"postgres-database":{"port":5432}},` +
|
|
`"grants":{"postgres-database":"/var/lib/mesh-host/grants"},` +
|
|
`"receives":{"postgres-database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
|
|
// The consumer also writes a configuration file with a hole in it, which is how nearly every
|
|
// real program takes a credential: a sealed file is a password alone, and almost nothing reads
|
|
// one. The mesh cannot compose the document — it discarded the value — so the module supplies it
|
|
// with `${secret:...}` in it and the host, the only thing that sees both halves, fills it in.
|
|
// A slug, so its identity on a backend stays within an S3 access key's 20 characters (ADR 0049).
|
|
await must("anchor", `printf %s '{"module":"meshboard","version":"1","slug":"board",` +
|
|
`"requires":["postgres-database"],"contributes":{"postgres-database":{"name":"meshboard"}},` +
|
|
`"binds":{"postgres-database":"/etc/meshboard/database.json"},` +
|
|
`"secrets":{"postgres-database":"/etc/meshboard/database.password"},` +
|
|
`"resources":[{"id":"env","type":"file","path":"/etc/meshboard/database.env","mode":"0600",` +
|
|
`"content":"PGHOST=$\{bound:postgres-database:at\}\\nPGPORT=$\{bound:postgres-database:port\}\\n` +
|
|
`PGUSER=$\{bound:postgres-database:as\}\\nPGPASSWORD=$\{secret:postgres-database\}\\n"}]}' ` +
|
|
`> /tmp/app.json`);
|
|
await must("anchor", `docker cp /tmp/pg.json mesh-controller:/pg.json`);
|
|
await must("anchor", `docker cp /tmp/app.json mesh-controller:/app.json`);
|
|
await mesh("module add /pg.json");
|
|
await mesh("module add /app.json");
|
|
|
|
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
|
await mesh("overlay place laptop --site lab");
|
|
for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
|
|
await mesh("assign anchor a-store");
|
|
await mesh("assign laptop meshboard");
|
|
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
// Once. On a warm return the host is already running (see `before`); a second one would
|
|
// consume the same queue and apply the same declaration twice, concurrently.
|
|
await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
|
|
}
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
|
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
|
|
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
|
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
|
|
|
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
|
// Named after the machine *and* the module, because a consumer is both (novox/hq
|
|
// 04-ISSUES/022) — a node routinely runs several modules wanting one database.
|
|
const onProvider = (await must("anchor",
|
|
`cat /var/lib/mesh-host/grants/laptop.meshboard.secret`)).trim();
|
|
assert.ok(onConsumer.length >= 40, `the consumer's credential is ${onConsumer.length} characters`);
|
|
assert.equal(onConsumer, onProvider,
|
|
"the two ends hold different passwords, so nothing could ever authenticate");
|
|
|
|
// Only the machine it is for may read it.
|
|
assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.password`), /^600/);
|
|
|
|
// And the configuration with holes in it arrived filled. **This is the only place the two
|
|
// substitutions are proven against a real host**: the mesh fills what it knows in the clear
|
|
// before sending, the host opens the sealed value and fills the rest on the machine, and the
|
|
// two expressions that find the holes live in different repositories.
|
|
const filled = await must("laptop", `cat /etc/meshboard/database.env`);
|
|
assert.match(filled, /^PGPASSWORD=.+$/m, `the password was never put in:\n${filled}`);
|
|
assert.ok(filled.includes(`PGPASSWORD=${onConsumer}`),
|
|
`the file holds a different password from the credential file:\n${filled}`);
|
|
assert.match(filled, /^PGUSER=mesh_laptop_board$/m,
|
|
`the consumer was not told what name to present:\n${filled}`);
|
|
assert.match(filled, /^PGPORT=5432$/m, `the port did not arrive as a port:\n${filled}`);
|
|
assert.doesNotMatch(filled, /\$\{/,
|
|
`a placeholder survived to the machine and would be read as a value:\n${filled}`);
|
|
assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.env`), /^600/);
|
|
|
|
// The password is in that file and nowhere the mesh could read it — which is the whole point of
|
|
// filling the hole on the machine rather than composing the document in the control plane.
|
|
|
|
// And it is nowhere it could have been read on the way. The declaration crossed the broker; the
|
|
// database is the control plane's; the state is what the node reported back.
|
|
for (const [machine, where] of [
|
|
["laptop", "/var/lib/mesh-host/declared.json"],
|
|
["laptop", "/var/lib/mesh-host/state.json"],
|
|
["anchor", "/var/lib/mesh-host/declared.json"],
|
|
] as const) {
|
|
// **`--` first, or the password is read as options.** A generated credential is random, so
|
|
// one of them eventually begins with a dash — this one started `-S` and grep refused the
|
|
// whole invocation. The test then compared an error message against "0" and reported the
|
|
// password as leaked, which is the worst way for a search to fail: it says it found
|
|
// something.
|
|
const { out } = await on(machine, `grep -c -e ${quote(onConsumer)} -- ${where}`);
|
|
assert.equal(out.trim(), "0", `the password is in ${where} on ${machine}`);
|
|
}
|
|
const inTheMesh = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select count(*) from secret where for_consumer like '%${onConsumer}%' ` +
|
|
`or for_provider like '%${onConsumer}%'"`);
|
|
assert.equal(inTheMesh.trim(), "0", "the control plane's database holds the password in the clear");
|
|
});
|
|
|
|
test("the consumer is also told where its database is", { skip, timeout: 300_000 }, async () => {
|
|
// A password with no address is not a connection. This is the readable half, which stays
|
|
// readable on purpose — it is the secret half that could not be composed, not this one.
|
|
const told = JSON.parse(await must("laptop", `cat /etc/meshboard/database.json`));
|
|
assert.equal(told.from, "anchor");
|
|
assert.equal(told.at, "anchor.internal");
|
|
assert.equal(told.serves.port, 5432);
|
|
// And the name it resolves to was written by the mesh as well, on this machine.
|
|
assert.match(await must("laptop", `grep anchor.internal /etc/hosts`), /10\.42\.0\.\d+/);
|
|
});
|
|
|
|
test("when a machine cannot do what it was told, the mesh says which and why", { skip, timeout: 900_000 }, async () => {
|
|
// Demonstrated with inserted rows first, which proves the query and not the path. This sends a
|
|
// real machine something it will genuinely fail at, and asks the mesh afterwards.
|
|
//
|
|
// A package that does not exist, because that is a failure of the ordinary kind: the host tries,
|
|
// the package manager says no, and some of the declaration is applied and some is not — which
|
|
// is the situation `status` exists to distinguish from a machine that refused everything.
|
|
await must("anchor", `printf %s '{"module":"impossible","version":"1","resources":[` +
|
|
`{"id":"nothing","type":"package","package":"a-package-that-does-not-exist"}]}' > /tmp/imp.json`);
|
|
await must("anchor", `docker cp /tmp/imp.json mesh-controller:/imp.json`);
|
|
await mesh("module add /imp.json");
|
|
await mesh("assign laptop impossible");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 10_000));
|
|
|
|
const said = await mesh("status");
|
|
assert.match(said, /not doing what they were told/, said);
|
|
assert.match(said, /laptop/, said);
|
|
// The machine's own words about the resource that failed, not a summary written at this end.
|
|
assert.match(said, /impossible\.nothing/, `the failing resource is not named:\n${said}`);
|
|
|
|
// And the distinction survives: this machine FAILED, it did not refuse. Refused means it is
|
|
// exactly as it was; failed means it is in a state nobody declared, and they are fixed in
|
|
// different places.
|
|
assert.match(said, /laptop\s+failed/, said);
|
|
|
|
// The other machine is not implicated.
|
|
assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/,
|
|
"a machine that did as it was told is listed as wrong");
|
|
});
|
|
|
|
test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => {
|
|
// A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so
|
|
// a push to one that is not listening must wait rather than vanish. The queue is durable and the
|
|
// message persistent, which ought to be enough — but a lost declaration is silent, and "ought to
|
|
// be" is not a property.
|
|
//
|
|
// The machine is not merely idle here: its host is stopped, so nothing is consuming its queue.
|
|
|
|
// Nothing this test asserts should depend on what another left behind. The machine still has a
|
|
// deliberately-impossible module from the test above, and while that is assigned the mesh never
|
|
// updates its account of what the machine holds — a partial report is not an account, on
|
|
// purpose (novox/hq 04-ISSUES/010).
|
|
await mesh("unassign laptop impossible");
|
|
|
|
// Stop listening, and prove it stopped — a test that pushed to a machine that was still running
|
|
// would pass having checked nothing.
|
|
//
|
|
// By process name, never by matching the command line: `pkill -f` matches the shell running it
|
|
// too, which kills the connection carrying the command and hangs the caller waiting for a reply
|
|
// that will never come. Cost an hour once, in this file.
|
|
await must("laptop", `pkill -x mesh-host || true; sleep 1`);
|
|
const listening = await on("laptop", `pgrep -x mesh-host`);
|
|
assert.equal(listening.ok, false, "the host is still running, so this proves nothing");
|
|
|
|
await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`);
|
|
await must("anchor", `docker cp /tmp/away.json mesh-controller:/away.json`);
|
|
await mesh("module add /away.json");
|
|
await mesh("assign laptop while-away");
|
|
await mesh("push laptop");
|
|
|
|
// Nothing has happened on the machine, because nothing is there to do it.
|
|
const before = await on("laptop", `test -f /etc/mesh-while-away`);
|
|
assert.equal(before.ok, false, "a machine with no host applied a declaration");
|
|
|
|
// And now it listens again. No second push, and nobody says anything.
|
|
await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`);
|
|
let arrived = false;
|
|
for (let i = 0; i < 20 && !arrived; i++) {
|
|
arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok;
|
|
if (!arrived) await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
if (!arrived) {
|
|
// Everything needed to tell "the message was never queued" from "the host never read it".
|
|
const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
|
|
const queues = await on("anchor",
|
|
`docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
|
|
const owned = await on("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
|
|
assert.fail(`a declaration sent to a switched-off machine was lost\n` +
|
|
`--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
|
|
`--- what each machine last did ---\n${owned.out}`);
|
|
}
|
|
assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
|
|
|
|
// And the mesh's account of what that machine holds catches up too, or a later declaration
|
|
// would tell it to remove what it has just been given.
|
|
await new Promise((r) => setTimeout(r, 4000));
|
|
const owned = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select owned from node where name = 'laptop'"`);
|
|
assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`);
|
|
});
|
|
|
|
test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => {
|
|
// Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares,
|
|
// and never what the machine raised for itself from its bundle — which is the fault that
|
|
// destroyed a foundation once (novox/hq 04-ISSUES/010).
|
|
//
|
|
// Two modules, so the test can tell "removed the right one" from "removed everything".
|
|
for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) {
|
|
await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`);
|
|
await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
|
await mesh(`module add /${name}.json`);
|
|
await mesh(`assign anchor ${name}`);
|
|
}
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive");
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive");
|
|
|
|
await mesh("unassign anchor going");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
|
|
assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false,
|
|
"an unassigned module's file is still there");
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok,
|
|
"unassigning one module took another one's file with it");
|
|
|
|
// And the foundation this machine raised from its own bundle is untouched. It was not declared by
|
|
// the mesh, so the mesh must never remove it — the machine would take its own control plane
|
|
// away, which is exactly what happened before origins existed.
|
|
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
for (const container of ["mesh-store", "mesh-broker", "mesh-controller"]) {
|
|
assert.match(running, new RegExp(container),
|
|
`${container} was removed by a declaration that never declared it`);
|
|
}
|
|
});
|
|
|
|
test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => {
|
|
// The other direction of the same rule. A node that is sent a declaration mentioning none of its
|
|
// private network must not lose it: the network came from a module that is still assigned, and
|
|
// "not in this message" is not "no longer wanted".
|
|
assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok,
|
|
"the private network's configuration is gone");
|
|
assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok,
|
|
"the mesh's names are gone");
|
|
});
|
|
|
|
test("a machine that fell behind catches up without being named", { skip, timeout: 900_000 }, async () => {
|
|
// `status` says which machines are not doing what they were told; something has to act on it.
|
|
// `push --behind` is that something, and it is a command rather than a timer to begin with —
|
|
// a scheduler is then a scheduler over this, rather than a second path to the same act.
|
|
|
|
// Break one machine, in a way that is fixable: a package that does not exist yet.
|
|
await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
|
|
`{"id":"pkg","type":"package","package":"a-package-that-does-not-exist-yet"},` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
|
|
await must("anchor", `docker cp /tmp/fix.json mesh-controller:/fix.json`);
|
|
await mesh("module add /fix.json");
|
|
await mesh("assign laptop fixable");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
assert.match(await mesh("status"), /laptop\s+failed/, "the machine did not report a failure");
|
|
// And the resources that COULD be applied were — one broken thing no longer blocks the rest
|
|
// (novox/hq 04-ISSUES/011).
|
|
assert.ok((await on("laptop", `test -f /etc/mesh-fixable`)).ok,
|
|
"a resource after the failing one was never attempted");
|
|
|
|
// Nothing is behind on the other machine, so nothing is pushed to it.
|
|
const named = await mesh("push --behind");
|
|
assert.match(named, /laptop/, named);
|
|
assert.doesNotMatch(named, /sent anchor/, `a machine that was fine was pushed to:\n${named}`);
|
|
|
|
// Fix the cause, the way somebody would: the module stops asking for the impossible thing.
|
|
await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
|
|
await must("anchor", `docker cp /tmp/fix.json mesh-controller:/fix.json`);
|
|
await mesh("module add /fix.json");
|
|
|
|
// And nobody names the machine.
|
|
await mesh("push --behind");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
const after = await mesh("status");
|
|
assert.doesNotMatch(after, /laptop\s+(failed|refused)/,
|
|
`the machine did not recover:\n${after}`);
|
|
|
|
// With nothing behind, it says so rather than doing nothing quietly.
|
|
assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
|
|
});
|
|
|
|
test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async () => {
|
|
// Artifacts go to a registry, and a mesh had no way to run its own — the only one that existed
|
|
// was raised by the lab, which is to say it existed nowhere but here.
|
|
//
|
|
// **Named, not mirrored** (novox/hq 04-ISSUES/029). Mirroring publishes to the artifact store,
|
|
// and the builder will not start without one — so a module that provides the store and builds
|
|
// its own image asks the mesh to put an artifact into the thing that artifact is needed to
|
|
// create. It used to pass here only because the LAB's registry was already standing to receive
|
|
// the push, which is exactly why a real first mesh would have found this and the lab did not.
|
|
//
|
|
// So the image is named by digest, the way the bundle names the three a first node starts from.
|
|
// A registry is the one module that cannot be delivered by the mesh's own delivery.
|
|
await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
|
|
`"provides":[{"name":"artifact-store","scope":"mesh"}],` +
|
|
`"capabilities":["container-runtime"],` +
|
|
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
|
|
`"serves":{"artifact-store":{"port":5000}},` +
|
|
`"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
|
|
`"resources":[` +
|
|
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
|
|
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
|
|
`"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
|
|
`> /root/registry/module.json`);
|
|
// **Added, not built** — and this is the half that proves the fix. Building needs a builder,
|
|
// and a builder will not start without an artifact store to publish to, so a mesh that has just
|
|
// bootstrapped cannot build the module that gives it one. Adding the manifest directly is the
|
|
// path a real first mesh has to take, so it is the path this walks.
|
|
await must("anchor", `docker cp /root/registry/module.json mesh-controller:/registry.json`);
|
|
await mesh("module add /registry.json");
|
|
await mesh("assign anchor registry");
|
|
await mesh("push anchor");
|
|
// The store's image is pulled from upstream at apply, over the uplink; that takes what it takes.
|
|
let names = "";
|
|
for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
names = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
}
|
|
// Running, and answering — a container that is up is not a registry that replies.
|
|
assert.match(names, /mesh-registry/,
|
|
`the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
|
|
let answers = false;
|
|
for (let i = 0; i < 20 && !answers; i++) {
|
|
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
|
|
if (!answers) await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
assert.ok(answers, "the mesh's own registry is running and does not answer");
|
|
|
|
// And reachable from another machine over the private network, which is the whole point of an
|
|
// artifact store being a mesh-scoped provision.
|
|
assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
|
|
"the artifact store is not reachable from another machine, so nothing else can use it");
|
|
});
|
|
|
|
// Defends novox/hq ADR 0007: the mesh is its own certificate authority for internal names.
|
|
test("a machine serves its internal name with a certificate the mesh issued", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
|
|
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
|
|
// moment something connects, which is the worst place to find out.
|
|
// The port the handshake below is tried on, declared: the anchor filters what its modules
|
|
// did not declare (ADR 0088), and a test server on an undeclared port proves only that.
|
|
await must("anchor", `printf %s '{"module":"served","version":"1",` +
|
|
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
|
|
`"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
|
|
`> /tmp/served.json`);
|
|
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
|
|
await mesh("module add /served.json");
|
|
await mesh("assign anchor served");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
|
|
assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");
|
|
|
|
// The name it was issued for is the one the mesh gave this machine.
|
|
const named = await must("anchor",
|
|
`openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
|
|
`docker run --rm -v /etc/mesh:/m ${ARTIFACT_STORE} sh -c ` +
|
|
`"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
|
|
assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);
|
|
|
|
// And a real handshake: the machine serves TLS with the key it generated, and another machine
|
|
// verifies it against the mesh's authority and nothing else.
|
|
await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
|
|
`-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
|
|
`> /var/log/tls.log 2>&1 & sleep 2`);
|
|
await must("laptop", `mkdir -p /etc/mesh`);
|
|
const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
|
|
await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);
|
|
|
|
const shook = await on("laptop",
|
|
`echo | openssl s_client -connect anchor.internal:8443 ` +
|
|
`-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
|
|
assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` +
|
|
`what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`);
|
|
assert.match(shook.out, /Verification: OK/, shook.out);
|
|
});
|
|
|
|
// Defends novox/hq ADR 0007: what a machine exposes is what its modules declared, and nothing
|
|
// arrives at a port nobody asked for.
|
|
test("a machine filters exactly what its modules declared, and nothing else", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// The rule set is derived from what is assigned, not kept in step by hand — and the proof that
|
|
// matters is not that a file arrived but that packets are treated differently because of it.
|
|
// A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
|
|
//
|
|
// Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
|
|
// so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
|
|
// the shape that rule leaves, and this is the first thing to use it for its real purpose.
|
|
// A listener is written to a file rather than squeezed through three levels of shell quoting.
|
|
// The first attempt did the latter, never started, and the test failed on its own setup —
|
|
// which reads exactly like the firewall working.
|
|
await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` +
|
|
`import socket, sys, threading\n` +
|
|
`def serve(port):\n` +
|
|
` s = socket.socket()\n` +
|
|
` s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` +
|
|
` s.bind(("0.0.0.0", port))\n` +
|
|
` s.listen(8)\n` +
|
|
` while True:\n` +
|
|
` c, _ = s.accept()\n` +
|
|
` c.send(str(port).encode())\n` +
|
|
` c.close()\n` +
|
|
`for p in (9101, 9102):\n` +
|
|
` threading.Thread(target=serve, args=(p,), daemon=True).start()\n` +
|
|
`threading.Event().wait()\n` +
|
|
`LISTENER`);
|
|
await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`);
|
|
|
|
// Two paths to the same machine, which is what makes "from the mesh" testable at all: over the
|
|
// private network, and over the segment both machines happen to share. A rule that opens a port
|
|
// to the mesh must accept the first and refuse the second — and a test that only ever used one
|
|
// path could not tell "open to the mesh" from "open".
|
|
const reach = async (where: string, port: number) => {
|
|
const said = await on("anchor",
|
|
`timeout 5 python3 -c "import socket;s=socket.create_connection(('${where}',${port}),4);` +
|
|
`print(s.recv(32).decode());s.close()"`);
|
|
return said.ok;
|
|
};
|
|
const overlay = (port: number) => reach("laptop.internal", port);
|
|
const segment = (port: number) => reach("192.0.2.20", port);
|
|
|
|
// Reachable both ways before any rule set exists, so what changes afterwards is the rule set and
|
|
// not the listener. Without this the test would pass against a service that never started.
|
|
assert.ok(await overlay(9101), "the declared port never opened, so nothing below tests anything");
|
|
assert.ok(await overlay(9102), "the undeclared port never opened");
|
|
assert.ok(await segment(9101), "the declared port is not reachable off the private network yet, " +
|
|
"so closing it later would prove nothing");
|
|
|
|
await must("anchor", `printf %s '{"module":"talker","version":"1",` +
|
|
`"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
|
|
`"resources":[]}' > /tmp/talker.json`);
|
|
// The rule set goes where this machine's nftables unit reads from, and the unit is declared to
|
|
// reflect it. No command anywhere.
|
|
// The module ships the unit that loads its rules, rather than using the one the distribution's
|
|
// nftables package provides. That unit is `Type=oneshot` with no `RemainAfterExit`, so it does
|
|
// its work and goes inactive — and a host asked for a service that is "running" reports, quite
|
|
// correctly, that it is stopped. There is no state in the vocabulary for "ran and exited having
|
|
// done its job", so a module that wants one brings a unit that stays.
|
|
//
|
|
// Which is also the right shape: how a machine enforces rules is a fact about the machine, and
|
|
// the mesh has no business depending on what a distribution happens to package.
|
|
await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
|
|
`"capabilities":["firewall"],` +
|
|
`"filtering":{"into":"/etc/mesh/filter.nft"},` +
|
|
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
|
|
`{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"},` +
|
|
`{"id":"unit","type":"file","path":"/etc/systemd/system/mesh-filter.service",` +
|
|
`"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for this machine\\n` +
|
|
`[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
|
|
`ExecStart=/usr/bin/nft -f /etc/mesh/filter.nft\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
|
|
`{"id":"filter","type":"service","unit":"mesh-filter.service","state":"running",` +
|
|
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
|
|
for (const f of ["talker", "firewall"]) {
|
|
await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
|
|
await mesh(`module add /${f}.json`);
|
|
}
|
|
await mesh("assign laptop talker");
|
|
await mesh("assign laptop firewall");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 20_000));
|
|
|
|
const written = await must("laptop", `cat /etc/mesh/filter.nft`);
|
|
// A rule names its source. Not decoration: it is the only thing that answers "why is this open".
|
|
assert.match(written, /# talker . the thing this test is about/,
|
|
`the rule does not name what caused it:\n${written}`);
|
|
// The mesh's addresses are the ones on the private network, which is what "from the mesh"
|
|
// means — not the segment the machines happen to share.
|
|
assert.match(written, /ip saddr \{ [0-9., ]+ \} tcp dport 9101 accept/,
|
|
`"from the mesh" resolved to nothing:\n${written}`);
|
|
assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);
|
|
|
|
// Loaded, not merely written. The service was restarted because a file it reflects changed.
|
|
const table = await must("laptop", `nft list table inet mesh`);
|
|
assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);
|
|
|
|
// And it filters. Three assertions, and the third is the one that makes "from the mesh" mean
|
|
// something rather than being a synonym for "open".
|
|
assert.ok(await overlay(9101),
|
|
"the declared port is closed on the private network, so the machine is filtering more than " +
|
|
"it was told to");
|
|
assert.ok(!(await overlay(9102)),
|
|
"a port no module declared is still reachable, so the rule set restricts nothing");
|
|
assert.ok(!(await segment(9101)),
|
|
"the declared port answers off the private network, so `from: mesh` restricted nothing");
|
|
|
|
// The machine did not lock itself out of the mesh: it is still taking declarations.
|
|
assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
|
|
"the machine stopped doing what it was told after applying its own rule set");
|
|
|
|
// Removing the module that wanted the port closes it, with nobody editing a rule. This is the
|
|
// whole claim of a derived firewall, and it is also the second load — which must replace the
|
|
// table rather than add to it.
|
|
await mesh("unassign laptop talker");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 20_000));
|
|
assert.ok(!(await overlay(9101)),
|
|
"the port stayed open after the module that wanted it was removed");
|
|
});
|
|
|
|
// The builder as a module the mesh assigns, with a credential the mesh delivered, is what genesis
|
|
// proves now (genesis-single installs the catalogue's builder through the installer, novox/hq ADR
|
|
// 0069). The test that lived here declared the builder's image as an upstream artifact by the bare
|
|
// image ID the lab holds, which is not a reference a registry copy can fetch (ADR 0096); retired
|
|
// 2026-09-21 rather than rewritten into a second genesis.
|
|
|
|
test("rotating a credential moves both ends, and the old one stops working", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: on
|
|
// 2026-08-22 a provision documented as never rotating minted a new password on every adoption
|
|
// and updated only the provider's row. Consumers on three nodes held dead credentials for two
|
|
// days while the mesh reported success.
|
|
//
|
|
// So this is checked against a real database with a real login, three times: the delivered
|
|
// credential works, the rotated one works, and the one that was rotated away does not. Two ends
|
|
// holding a matching string proves they agree; only an authentication proves they are right.
|
|
const store = "/var/lib/mesh/postgres";
|
|
await must("anchor", `printf %s '{"module":"realstore","version":"1",` +
|
|
`"provides":[{"name":"real-postgres-database","scope":"mesh"}],` +
|
|
`"capabilities":["container-runtime"],` +
|
|
`"serves":{"real-postgres-database":{"port":5433}},` +
|
|
`"own-secrets":{"superuser":"${store}/superuser"},` +
|
|
`"grants":{"real-postgres-database":"${store}/grants"},` +
|
|
// Both halves. `grants` is where each consumer's sealed password lands; `receives` is the
|
|
// manifest saying who asked and for what. Without the second the provisioner finds a
|
|
// directory of unexplained secrets and says nothing has been granted — which is true, and
|
|
// reads exactly like a credential that was never delivered.
|
|
`"receives":{"real-postgres-database":"${store}/grants/mesh.json"},` +
|
|
`"listens":[{"port":5433,"from":"mesh","why":"a database the mesh provisions"}],` +
|
|
`"resources":[` +
|
|
`{"id":"state","type":"directory","path":"${store}","mode":"0755"},` +
|
|
`{"id":"grants","type":"directory","path":"${store}/grants","mode":"0755"},` +
|
|
`{"id":"postgres-database","type":"container","name":"real-store",` +
|
|
`"image":"${pinned("postgres")}",` +
|
|
`"ports":["5433:5432"],` +
|
|
`"volumes":["${store}/superuser:/run/superuser:ro"],` +
|
|
`"env":{"POSTGRES_PASSWORD_FILE":"/run/superuser"}},` +
|
|
`{"id":"provisioner","type":"container","name":"real-provisioner",` +
|
|
`"image":"${pinned("mesh-provision-postgres")}","network":"host",` +
|
|
`"volumes":["${store}:${store}:ro"],` +
|
|
`"env":{"GRANTS":"${store}/grants",` +
|
|
`"MESH_PROVISION_PASSWORD_FILE":"${store}/superuser",` +
|
|
`"MESH_PROVISION_POSTGRES":"postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable"}}]}' ` +
|
|
`> /tmp/realstore.json`);
|
|
await must("anchor", `printf %s '{"module":"realapp","version":"1",` +
|
|
`"requires":["real-postgres-database"],"contributes":{"real-postgres-database":{"name":"realapp"}},` +
|
|
`"binds":{"real-postgres-database":"/etc/realapp/where.json"},` +
|
|
`"secrets":{"real-postgres-database":"/etc/realapp/password"},` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/realapp","mode":"0755"}]}' ` +
|
|
`> /tmp/realapp.json`);
|
|
for (const f of ["realstore", "realapp"]) {
|
|
await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
|
|
await mesh(`module add /${f}.json`);
|
|
}
|
|
await mesh("assign anchor realstore");
|
|
await mesh("assign laptop realapp");
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 30_000));
|
|
|
|
// A real login from the consumer's machine, over the private network — not over loopback, where
|
|
// pg_hba trusts anything and every password looks correct. That was done here once and the test
|
|
// passed for an afternoon while verifying nothing: a deliberately wrong password returned a row.
|
|
// As the role the provisioner made, into the database it made. The provisioner names a role
|
|
// after the machine and a database after what the module asked for — which is the contract, and
|
|
// getting it wrong here made the test fail against a provisioner that had done its job.
|
|
// By address, resolved on the machine.
|
|
//
|
|
// **This container is not the mesh's.** The mesh gives its names to the containers it declares,
|
|
// and this one is started by the test with `docker run` — nothing declared it, so nothing
|
|
// configured it. That boundary is the right one: a container somebody runs by hand is not the
|
|
// mesh's to configure, and reaching into every container on a machine is what a resolver in
|
|
// resolv.conf would be for.
|
|
//
|
|
// So the workaround stays here, and the proof that names work inside containers is its own
|
|
// test, against a container the mesh declared.
|
|
const where = (await must("laptop",
|
|
`getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
|
|
assert.match(where, /^[0-9.]+$/, `the mesh's name for anchor does not resolve here: ${where}`);
|
|
|
|
const login = async (password: string) =>
|
|
await on("laptop", `docker run --rm -e PGPASSWORD=${quote(password)} ` +
|
|
// The role the provisioner made: mesh_<node>_<module>, because a consumer is a module on
|
|
// a machine (novox/hq 04-ISSUES/022).
|
|
`${pinned("postgres")} psql -h ${where} -p 5433 -U mesh_laptop_realapp ` +
|
|
`-d realapp -qAt -c "select 1"`, 120_000);
|
|
|
|
const diagnostics = async () =>
|
|
`provisioner:\n${(await on("anchor", `docker logs real-provisioner 2>&1 | tail -20`)).out}\n` +
|
|
`grants:\n${(await on("anchor", `ls -l ${store}/grants`)).out}`;
|
|
|
|
const first = (await must("laptop", `cat /etc/realapp/password`)).trim();
|
|
assert.ok(first.length >= 40, `the consumer's credential is ${first.length} characters`);
|
|
let works = false;
|
|
for (let i = 0; i < 20 && !works; i++) {
|
|
works = (await login(first)).ok;
|
|
if (!works) await new Promise((r) => setTimeout(r, 5000));
|
|
}
|
|
assert.ok(works, `the delivered credential does not authenticate:\n` +
|
|
`${(await login(first)).out}\n${await diagnostics()}`);
|
|
|
|
// Now rotate. One command: the record changes AND both ends are sent, because leaving the
|
|
// sending to a later command is the fault above, exactly.
|
|
const said = await mesh("rotate real-postgres-database", 180_000);
|
|
assert.match(said, /anchor/, `rotation did not touch the provider:\n${said}`);
|
|
assert.match(said, /laptop/, `rotation did not touch the consumer:\n${said}`);
|
|
await new Promise((r) => setTimeout(r, 25_000));
|
|
|
|
const second = (await must("laptop", `cat /etc/realapp/password`)).trim();
|
|
assert.notEqual(second, first, "the consumer was handed back the credential just rotated away");
|
|
|
|
// The new one authenticates — the only proof the provider was told the same thing the consumer
|
|
// was given. Two files agreeing proves they agree, not that either is right.
|
|
let now = false;
|
|
for (let i = 0; i < 20 && !now; i++) {
|
|
now = (await login(second)).ok;
|
|
if (!now) await new Promise((r) => setTimeout(r, 5000));
|
|
}
|
|
assert.ok(now, `after rotation the new credential does not authenticate, so the two ends ` +
|
|
`disagree — which is the fault this exists to make impossible:\n${await diagnostics()}`);
|
|
|
|
// And the old one does not. Without this the test passes against a provider that added a
|
|
// password without replacing one, which is a rotation that rotates nothing.
|
|
assert.ok(!(await login(first)).ok,
|
|
"the password that was rotated away still authenticates, so nothing was rotated");
|
|
});
|
|
|
|
test("a route is a grant: a workload is reached by the name it asked for", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// novox/hq 08-connectivity §3. The mirror of a database grant: there the consumer supplies a
|
|
// name and receives credentials; here it supplies a target and receives a name. Nothing new in
|
|
// the vocabulary — a route is a provision like any other.
|
|
//
|
|
// The workload is the registry image, because it is an HTTP server this scenario already has.
|
|
// What is being tested is the mesh's arrangement, not the workload.
|
|
await must("anchor", `printf %s '{"module":"frontdoor","version":"1",` +
|
|
`"provides":[{"name":"route","scope":"mesh"}],` +
|
|
`"capabilities":["container-runtime"],` +
|
|
`"receives":{"route":"/etc/frontdoor/routes.json"},` +
|
|
`"serves":{"route":{"domain":"mesh.test"}},` +
|
|
`"listens":[{"port":8081,"from":"mesh","why":"the front door"}],` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/frontdoor","mode":"0755"},` +
|
|
`{"id":"proxy","type":"container","name":"front-door",` +
|
|
`"image":"${pinned("mesh-route-proxy")}","network":"host",` +
|
|
`"volumes":["/etc/frontdoor:/etc/frontdoor:ro"],` +
|
|
`"env":{"ROUTES":"/etc/frontdoor/routes.json","LISTEN":":8081"}}]}' ` +
|
|
`> /tmp/frontdoor.json`);
|
|
// The workload declares the port it listens on as well as the route it wants. Both, because
|
|
// they are different questions: one says who may reach it, the other says by what name — and
|
|
// the earlier test left this machine filtering, so a module that asked for a route and not for
|
|
// the port would be unreachable by the proxy it just asked for.
|
|
await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` +
|
|
`"requires":["route"],"capabilities":["container-runtime"],` +
|
|
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
|
|
`"binds":{"route":"/etc/storefront/route.json"},` +
|
|
`"listens":[{"port":8088,"from":"mesh","why":"the proxy reaches it here"}],` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/storefront","mode":"0755"},` +
|
|
`{"id":"app","type":"container","name":"storefront",` +
|
|
`"image":"${ARTIFACT_STORE}","ports":["8088:5000"]}]}' > /tmp/storefront.json`);
|
|
for (const f of ["frontdoor", "storefront"]) {
|
|
await must("anchor", `docker cp /tmp/${f}.json mesh-controller:/${f}.json`);
|
|
await mesh(`module add /${f}.json`);
|
|
}
|
|
await mesh("assign anchor frontdoor");
|
|
await mesh("assign laptop storefront");
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 25_000));
|
|
|
|
// The provider was told who asked, and where that machine is — which it needs in order to
|
|
// reach back, and which it must not have to derive from a naming convention.
|
|
const routes = await must("anchor", `cat /etc/frontdoor/routes.json`);
|
|
assert.match(routes, /shop\.mesh\.test/, `the proxy was not told about the route:\n${routes}`);
|
|
assert.match(routes, /"at": *"laptop\.internal"/,
|
|
`the proxy was not told where the consumer is, so it cannot reach it:\n${routes}`);
|
|
|
|
// And the consumer was told what the provider serves, which is how it knows its own name.
|
|
const bound = await must("laptop", `cat /etc/storefront/route.json`);
|
|
assert.match(bound, /mesh\.test/, `the consumer was not told the public name:\n${bound}`);
|
|
|
|
// The whole point: a request for the name reaches the workload, across the private network.
|
|
let reached = false;
|
|
let said = "";
|
|
for (let i = 0; i < 20 && !reached; i++) {
|
|
const answer = await on("anchor",
|
|
`curl -sf -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
|
|
said = answer.out;
|
|
reached = answer.ok && said.trim() === "200";
|
|
if (!reached) await new Promise((r) => setTimeout(r, 4000));
|
|
}
|
|
assert.ok(reached, `a request for the name did not reach the workload (${said}):\n` +
|
|
`${(await on("anchor", `docker logs front-door 2>&1 | tail -20`)).out}`);
|
|
|
|
// Withdrawal, which 08-connectivity lists as open: a stale public name pointing at nothing
|
|
// fails more visibly than a stale grant, so it must not survive the module leaving.
|
|
await mesh("unassign laptop storefront");
|
|
await mesh("push");
|
|
|
|
// **Waited for, not slept through.** The mesh withdrawing a route and the machine acting on it
|
|
// are different things, and a fixed sleep between them tests whichever the clock happened to
|
|
// land on — this assertion passed twice and failed once on nothing but timing.
|
|
//
|
|
// A machine that has not applied yet is also not a machine that failed, so `status` cannot
|
|
// stand in for this: "not yet" and "never" look identical there, and only one of them is worth
|
|
// failing over.
|
|
let after = "";
|
|
let withdrawn = false;
|
|
for (let i = 0; i < 20 && !withdrawn; i++) {
|
|
after = await must("anchor", `cat /etc/frontdoor/routes.json`);
|
|
withdrawn = !after.includes("shop.mesh.test");
|
|
if (!withdrawn) await new Promise((r) => setTimeout(r, 3000));
|
|
}
|
|
assert.ok(withdrawn,
|
|
`the route outlived the module that asked for it:\n${after}\n\n` +
|
|
`the machine did apply — this is what the mesh would send now:\n` +
|
|
`${(await on("anchor", `docker exec mesh-controller /mesh-controller plan anchor --files`)).out}`);
|
|
|
|
let gone = false;
|
|
for (let i = 0; i < 15 && !gone; i++) {
|
|
const answer = await on("anchor",
|
|
`curl -s -H 'Host: shop.mesh.test' http://127.0.0.1:8081/v2/ -o /dev/null -w '%{http_code}'`);
|
|
gone = answer.out.trim() === "404";
|
|
if (!gone) await new Promise((r) => setTimeout(r, 3000));
|
|
}
|
|
assert.ok(gone, "the proxy still serves a name whose module was unassigned");
|
|
});
|
|
|
|
test("model access is answered by a record, and the key the mesh took is one it cannot read", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// novox/hq ADR 0024. The first provision no machine answers: a hosted model is on nobody's
|
|
// node and is reached over the public internet, so the rule that refuses two ends sharing no
|
|
// private network must not apply to it.
|
|
await must("anchor", `printf %s '{"module":"assistant","version":"1",` +
|
|
`"requires":["model-access"],` +
|
|
`"binds":{"model-access":"/etc/assistant/model.json"},` +
|
|
`"secrets":{"model-access":"/etc/assistant/key"},` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/assistant","mode":"0755"}]}' ` +
|
|
`> /tmp/assistant.json`);
|
|
await must("anchor", `docker cp /tmp/assistant.json mesh-controller:/assistant.json`);
|
|
await mesh("module add /assistant.json");
|
|
|
|
// The licences first. With none recorded at all the honest answer is that nothing provides
|
|
// model-access — correct, and a different refusal from the one being tested.
|
|
await mesh(`licence add anthropic personal --serves '{"model":"a-model"}'`);
|
|
await mesh(`licence add anthropic the-organisation --serves '{"model":"a-model"}'`);
|
|
|
|
// Refused at the earliest point somebody could meet it: assigning records the assignment and
|
|
// then says the machine's set cannot be applied. The refusal names both candidates and the
|
|
// command. ADR 0024 warns this will be felt — which is correct, and correct is not the same as
|
|
// usable.
|
|
const refused = await on("anchor", `docker exec mesh-controller /mesh-controller assign laptop assistant`);
|
|
assert.ok(!refused.ok,
|
|
`a consumer was given model access without anybody saying which:\n${refused.out}`);
|
|
for (const want of ["personal", "the-organisation", "licence use"]) {
|
|
assert.match(refused.out, new RegExp(want),
|
|
`the refusal does not name ${want}:\n${refused.out}`);
|
|
}
|
|
|
|
await mesh("licence use personal laptop assistant");
|
|
|
|
// Chosen, and still no key: the mesh has one thing to deliver and has not been given it.
|
|
const noKey = await on("anchor", `docker exec mesh-controller /mesh-controller plan laptop`);
|
|
assert.ok(!noKey.ok, `a module was planned with a licence that has no key:\n${noKey.out}`);
|
|
assert.match(noKey.out, /licence key personal/, noKey.out);
|
|
|
|
// The accept verb. Given on standard input rather than as an argument, because a key in a
|
|
// command line is a key in shell history and in every process listing taken while it ran.
|
|
const secret = "sk-test-" + "0123456789abcdef".repeat(2);
|
|
const accepted = await must("anchor",
|
|
`printf %s ${quote(secret)} | docker exec -i mesh-controller /mesh-controller licence key personal`);
|
|
assert.match(accepted, /sealed to 1 holder/, accepted);
|
|
assert.doesNotMatch(accepted, new RegExp(secret),
|
|
"the key was echoed back, so the one copy that matters is on a terminal");
|
|
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
|
|
// What is public arrives, and says it is a record rather than leaving an empty address that a
|
|
// reader would take for something the mesh failed to fill in.
|
|
const bound = await must("laptop", `cat /etc/assistant/model.json`);
|
|
assert.match(bound, /personal/, `the consumer was not told which licence it is on:\n${bound}`);
|
|
assert.match(bound, /a-model/, `what the licence serves did not arrive:\n${bound}`);
|
|
assert.match(bound, /not a machine/, `the binding leaves an unexplained empty address:\n${bound}`);
|
|
|
|
// And the key arrives, readable only by this machine.
|
|
assert.equal((await must("laptop", `cat /etc/assistant/key`)).trim(), secret,
|
|
"the key that arrived is not the key that was given");
|
|
assert.match(await must("laptop", `stat -c %a /etc/assistant/key`), /^600/);
|
|
|
|
// The mesh cannot read it back. This is the whole argument: what is stored is unusable by
|
|
// whoever holds it, the control plane included.
|
|
const stored = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d licences -qAt ` +
|
|
`-c "select coalesce(sealed,'') from licence_holder"`);
|
|
assert.ok(stored.trim().length > 0, "nothing was stored, so nothing was sealed");
|
|
assert.doesNotMatch(stored, new RegExp(secret),
|
|
"the key is in the control plane's own database in the open");
|
|
|
|
// Nor is it anywhere it could have been read on the way.
|
|
for (const where of ["/var/lib/mesh-host/declared.json", "/var/lib/mesh-host/state.json"]) {
|
|
// Whether grep found it, not how many times. `grep -c` prints 0 and exits non-zero when it
|
|
// finds nothing, so the obvious `|| echo 0` prints a second one and the count is never what
|
|
// it looks like.
|
|
const found = await on("laptop", `grep -q ${quote(secret)} ${where}`);
|
|
assert.ok(!found.ok, `the key is in the open in ${where}`);
|
|
}
|
|
});
|
|
|
|
test("a new commit reaches a machine that is already running the old one", {
|
|
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
|
|
timeout: 900_000,
|
|
}, async () => {
|
|
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
|
|
// question "did my change go out?". This is that question, end to end — a commit, a build, a
|
|
// catalogue, and a machine that ends up running what the source says.
|
|
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
|
|
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
|
|
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
|
|
await startBuilder();
|
|
const repo = "/var/lib/mesh/builder/repositories/delivered";
|
|
const write = async (what: string) =>
|
|
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
|
|
`"resources":[{"id":"marker","type":"file","path":"/etc/delivered",` +
|
|
`"content":"${what}","mode":"0644"}]}' > ${repo}/module.json`);
|
|
|
|
await write("first");
|
|
await must("anchor", `cd ${repo} && git init -q . && git add -A && ` +
|
|
`git -c user.email=lab -c user.name=lab commit -qm first`);
|
|
await mesh(`build ${repo} --wait 300s`, 420_000);
|
|
await mesh("assign laptop delivered");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first");
|
|
|
|
// Nothing has moved, so nothing is behind — and it says so rather than doing nothing quietly.
|
|
assert.match(await mesh("build --behind"), /every module the mesh holds is what its source last had/);
|
|
|
|
// Now the source moves.
|
|
await write("second");
|
|
await must("anchor", `cd ${repo} && git add -A && ` +
|
|
`git -c user.email=lab -c user.name=lab commit -qm second`);
|
|
const moved = (await must("anchor", `cd ${repo} && git rev-parse HEAD`)).trim();
|
|
await mesh(`module moved delivered ${moved}`);
|
|
|
|
// The mesh says which module is behind, and by how much, before anything is built.
|
|
const behind = await mesh("status");
|
|
assert.match(behind, /delivered/, `status does not name the module that moved:\n${behind}`);
|
|
assert.match(behind, /build --behind/, `status does not say how to catch up:\n${behind}`);
|
|
|
|
// The loop, in one command: everything behind its source is built and recorded.
|
|
const built = await mesh("build --behind --wait 300s", 420_000);
|
|
assert.match(built, /delivered/, built);
|
|
|
|
// The machine is still running the old one until it is told — the mesh changing its mind is
|
|
// not the same as a machine acting on it, and collapsing the two is how a mesh reports success
|
|
// for something that has not happened.
|
|
assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "first",
|
|
"the machine changed before anything was sent to it");
|
|
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
assert.equal((await must("laptop", `cat /etc/delivered`)).trim(), "second",
|
|
"the machine is still running what the source no longer says");
|
|
|
|
// And it is no longer out of date, which is the half that makes the answer trustworthy: a
|
|
// status that says "behind" for ever is one nobody reads.
|
|
const after = await mesh("status");
|
|
assert.doesNotMatch(after, /delivered.*<.*[0-9a-f]{8}/,
|
|
`the module is still reported as behind after catching up:\n${after}`);
|
|
});
|
|
|
|
test("the board names the machine that is not doing what it was told", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// novox/hq 03-DESIGN/01-to-be/11-a-board.md. The board being replaced reads every context's
|
|
// database directly; this one asks the same questions through the same functions the commands
|
|
// use, and holds nothing. So the check is that what it says matches what the mesh says, and
|
|
// that it says the thing a person opened it for.
|
|
await must("anchor", `printf %s '{"module":"board","version":"1",` +
|
|
`"listens":[{"port":8090,"from":"mesh","why":"the board"}],` +
|
|
`"resources":[]}' > /tmp/board.json`);
|
|
await must("anchor", `docker cp /tmp/board.json mesh-controller:/board.json`);
|
|
await mesh("module add /board.json");
|
|
|
|
// Served from the control plane's own container, reading the mesh on every request.
|
|
await must("anchor", `docker exec -d mesh-controller /mesh-controller board --listen 0.0.0.0:8090`);
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
|
|
const read = async (path: string) =>
|
|
await on("anchor", `curl -sf http://127.0.0.1:8090${path}`, 30_000);
|
|
|
|
let up = false;
|
|
let said = { out: "", ok: false };
|
|
for (let i = 0; i < 15 && !up; i++) {
|
|
said = await read("/");
|
|
up = said.ok;
|
|
if (!up) await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
assert.ok(up, `the board does not answer:\n${said.out}`);
|
|
|
|
// Something a person opened it for: give a machine a declaration it cannot apply.
|
|
//
|
|
// A unit that does not exist, because the host says so immediately and in its own words. A file
|
|
// in a missing directory is not impossible — the host creates the parents, which is correct and
|
|
// made the first version of this test break nothing at all.
|
|
await must("anchor", `printf %s '{"module":"impossible","version":"1",` +
|
|
`"resources":[{"id":"nowhere","type":"service","unit":"nothing-like-this.service",` +
|
|
`"state":"running"}]}' > /tmp/impossible.json`);
|
|
await must("anchor", `docker cp /tmp/impossible.json mesh-controller:/impossible.json`);
|
|
await mesh("module add /impossible.json");
|
|
await mesh("assign laptop impossible");
|
|
await mesh("push laptop");
|
|
|
|
let named = false;
|
|
let page = "";
|
|
for (let i = 0; i < 20 && !named; i++) {
|
|
page = (await read("/")).out;
|
|
named = page.includes("laptop") && page.includes(">failed<");
|
|
if (!named) await new Promise((r) => setTimeout(r, 3000));
|
|
}
|
|
assert.ok(named, `the board does not name the machine that failed:\n${page}`);
|
|
|
|
// The host's own words, which say exactly what it could not do. A board that said only
|
|
// "failed" would make a person go and ask the thing they opened the board to avoid asking.
|
|
assert.match(page, /nowhere/, `the board does not say what failed:\n${page}`);
|
|
|
|
// It agrees with the command, because both read the same thing. Two answers to "which machine
|
|
// is broken" is worse than either alone.
|
|
const asJSON = JSON.parse((await read("/mesh.json")).out);
|
|
assert.equal(asJSON.wrong[0].node, "laptop", `the page and the JSON disagree: ${JSON.stringify(asJSON)}`);
|
|
assert.equal(asJSON.wrong[0].outcome, "failed");
|
|
|
|
const fromCommand = JSON.parse(await mesh("status --json"));
|
|
assert.deepEqual(asJSON.wrong, fromCommand.wrong,
|
|
"the board and the command disagree about which machine is broken");
|
|
|
|
// And it changes nothing: the mesh is exactly as it was after being read.
|
|
const before = await mesh("status");
|
|
await read("/");
|
|
assert.equal(await mesh("status"), before, "reading the board changed the mesh");
|
|
|
|
await mesh("unassign laptop impossible");
|
|
await mesh("push laptop");
|
|
});
|
|
|
|
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
|
|
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on
|
|
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a
|
|
// filter module derives the rules — so the credential test above assigns the catalogue's and
|
|
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh
|
|
// was not severed. Retired 2026-09-22.
|
|
|
|
test("a container reaches another machine by the name the mesh gave it", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// Internal names are written to the machine's hosts file, which serves the machine and not what
|
|
// the machine runs: a container gets its own hosts file holding only its own hostname. So every
|
|
// name the mesh wrote was invisible to the majority of things that need one.
|
|
//
|
|
// Checked from inside a container rather than on the machine, because on the machine it has
|
|
// always worked — and that is exactly what made this easy to miss.
|
|
await must("anchor", `printf %s '{"module":"resolves","version":"1",` +
|
|
`"capabilities":["container-runtime"],` +
|
|
`"resources":[{"id":"idle","type":"container","name":"resolves",` +
|
|
`"image":"${ARTIFACT_STORE}"}]}' > /tmp/resolves.json`);
|
|
await must("anchor", `docker cp /tmp/resolves.json mesh-controller:/resolves.json`);
|
|
await mesh("module add /resolves.json");
|
|
await mesh("assign laptop resolves");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
|
|
// The names are in the container's own hosts file, written by the runtime.
|
|
const inside = await must("laptop", `docker exec resolves cat /etc/hosts`);
|
|
assert.match(inside, /anchor\.internal/,
|
|
`the container cannot see the other machine's name:\n${inside}`);
|
|
assert.match(inside, /laptop\.internal/,
|
|
`the container cannot see its own machine's name:\n${inside}`);
|
|
|
|
// And the name actually reaches the machine, which is the part that matters: a hosts entry
|
|
// pointing at the wrong address resolves perfectly and connects to nothing.
|
|
const reached = await on("laptop",
|
|
`docker exec resolves sh -c 'getent hosts anchor.internal'`);
|
|
assert.ok(reached.ok, `the name does not resolve inside the container:\n${reached.out}`);
|
|
const address = reached.out.trim().split(/\s+/)[0];
|
|
const onTheMachine = (await must("laptop",
|
|
`getent hosts anchor.internal | head -1 | cut -d' ' -f1`)).trim();
|
|
assert.equal(address, onTheMachine,
|
|
"the container and its machine disagree about where the other machine is");
|
|
|
|
await mesh("unassign laptop resolves");
|
|
await mesh("push laptop");
|
|
});
|
|
|
|
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
|
|
// told each one.
|
|
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
|
|
* it is issued once per machine. The mesh writes the resolver's data as a fact the module
|
|
* declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
|
|
const resolverIssued = new Set<string>();
|
|
async function resolverModules(machines: string[]): Promise<void> {
|
|
for (const name of ["dnsmasq", "resolved-split-dns"]) {
|
|
const manifest = catalogueModule(name, held);
|
|
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
|
|
await mesh(`module add /${name}.json`);
|
|
}
|
|
for (const machine of machines) {
|
|
if (resolverIssued.has(machine)) continue;
|
|
await mesh(`module issue dnsmasq --node ${machine}`);
|
|
resolverIssued.add(machine);
|
|
}
|
|
}
|
|
|
|
test("every name under a machine resolves to that machine", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// Services are named under the machine they run on — postgres.novox.internal,
|
|
// plex.ace.internal. The first label is the service and the rest is the node, so what must
|
|
// resolve is anything under a node's name. What routes it once it arrives is a proxy's, and
|
|
// stays separate.
|
|
//
|
|
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the
|
|
// mesh has no business choosing one. So what is checked here is the mesh's half — that the
|
|
// data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq
|
|
// declares, so that module is what is assigned; what it runs is the next test's concern.
|
|
await resolverModules(["anchor", "laptop"]);
|
|
await mesh("assign anchor dnsmasq");
|
|
await mesh("assign laptop dnsmasq");
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
const written = await must(machine, `cat /etc/mesh-resolver/nodes.conf`);
|
|
|
|
// A wildcard per machine, matching the name and everything under it. Both machines get the
|
|
// whole mesh: a node resolves every other node, and itself.
|
|
for (const node of ["anchor", "laptop"]) {
|
|
assert.match(written, new RegExp(`address=/${node}\\.internal/10\\.42\\.0\\.\\d+`),
|
|
`${machine} cannot resolve names under ${node}:\n${written}`);
|
|
}
|
|
|
|
// And the addresses agree with what the machine's own hosts file says. Two accounts of where
|
|
// a machine is, disagreeing, would be worse than either alone — and this is the one place
|
|
// they could drift, because they are generated separately.
|
|
const hosts = await must(machine, `getent hosts anchor.internal | head -1 | cut -d' ' -f1`);
|
|
assert.match(written, new RegExp(`address=/anchor\\.internal/${hosts.trim().replace(/\./g, "\\.")}`),
|
|
`the resolver data and the hosts file disagree about where anchor is:\n${written}`);
|
|
}
|
|
|
|
// It follows the machines. A node leaving the private network must stop being answered for,
|
|
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
|
|
// fails at once and says which name it was.
|
|
//
|
|
// Both: the resolver's data follows the private network, so the machine leaves the network as
|
|
// well as the resolver, and what is asserted is that the machine that stayed is answered for and
|
|
// the one that left is not.
|
|
await mesh("unassign laptop dnsmasq");
|
|
await mesh("unassign laptop networking");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
|
|
const after = await must("anchor", `cat /etc/mesh-resolver/nodes.conf`);
|
|
assert.doesNotMatch(after, /address=\/laptop\.internal\//,
|
|
`a machine that left the private network is still answered for:\n${after}`);
|
|
assert.match(after, /address=\/anchor\.internal\//,
|
|
`the machine that stayed lost its own name:\n${after}`);
|
|
|
|
await mesh("assign laptop networking");
|
|
await mesh("unassign anchor dnsmasq");
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 15_000));
|
|
});
|
|
|
|
test("a service is reached by a name under the machine it runs on", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
|
|
// the node, so anything under a node's name must resolve to that node. What routes it once it
|
|
// arrives is a proxy's concern and stays separate.
|
|
//
|
|
// The mesh writes the data; a module runs the daemon. Both manifests are read from the
|
|
// repository rather than written here, so what is proven is what ships.
|
|
// `resolved-split-dns`, not `resolv-conf`: these machines run systemd-resolved, which owns
|
|
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
|
|
// refusal rather than a fight over the file — and picking the wrong one here would have been
|
|
// testing that fight.
|
|
await resolverModules(["anchor", "laptop"]);
|
|
|
|
// Both machines, because a node resolves from its own copy — the same rule as everything else
|
|
// it holds. A mesh where one machine answers for all of them stops resolving when that machine
|
|
// does, which is the arrangement this design refuses everywhere else.
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
await mesh(`assign ${machine} dnsmasq`);
|
|
await mesh(`assign ${machine} resolved-split-dns`);
|
|
}
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 25_000));
|
|
|
|
// Everything this test could want to know, gathered in one place.
|
|
//
|
|
// Three times now a diagnostic has not run because the thing before it threw: `must` on a
|
|
// command that fails, and then a query that hangs long enough to take the harness's own timeout
|
|
// with it. A 30-second test costs fifteen minutes to re-run, so the evidence has to be gathered
|
|
// whether the failure is an assertion, an error, or a hang.
|
|
const diagnose = async (machine: string) =>
|
|
`--- ${machine}\n` +
|
|
`dnsmasq: ${(await on(machine, `systemctl is-active dnsmasq.service`)).out.trim()}\n` +
|
|
`${(await on(machine, `journalctl -u dnsmasq -n 12 --no-pager`)).out}\n` +
|
|
`resolved: ${(await on(machine, `resolvectl status | head -30`)).out}\n` +
|
|
`resolv.conf:\n${(await on(machine, `cat /etc/resolv.conf`)).out}\n` +
|
|
`what the mesh wrote:\n${(await on(machine, `cat /etc/mesh-resolver/nodes.conf`)).out}\n` +
|
|
`listening:\n${(await on(machine, `ss -lnup | grep :53 || echo none`)).out}\n` +
|
|
`asked directly:\n${(await on(machine,
|
|
`timeout 5 resolvectl query postgres.anchor.internal 2>&1 || echo "no answer"`)).out}`;
|
|
|
|
// `on`, not `must`: `is-active` exits non-zero for a unit that failed, so `must` would throw
|
|
// before the assertion below — taking every diagnostic with it. That happened, and the run said
|
|
// only "failed".
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
const state = await on(machine, `systemctl is-active dnsmasq.service`);
|
|
if (state.out.trim() === "active") continue;
|
|
assert.fail(`the resolver is not running on ${machine} (${state.out.trim()}):\n\n` +
|
|
`its config:\n${(await on(machine, `cat /etc/dnsmasq.conf`)).out}\n` +
|
|
`${await diagnose(machine)}`);
|
|
}
|
|
|
|
// Through the machine's own resolver, by the path an application actually takes: nsswitch, then
|
|
// files, then DNS. `dig` would ask a server directly and prove less — the resolv.conf module is
|
|
// half of what is being tested, and only this path goes through it.
|
|
//
|
|
// Bounded on the machine rather than by the harness: a query that hangs is a result, and letting
|
|
// it run into the harness's own timeout turns it into an error with no evidence attached.
|
|
const resolves = async (machine: string, name: string) => {
|
|
const said = await on(machine,
|
|
`timeout 5 getent hosts ${name} | head -1 | cut -d' ' -f1`, 20_000);
|
|
return said.out.trim();
|
|
};
|
|
const addressOf = async (machine: string, node: string) =>
|
|
(await must(machine, `getent hosts ${node}.internal | head -1 | cut -d' ' -f1`)).trim();
|
|
|
|
const anchorAt = await addressOf("anchor", "anchor");
|
|
const laptopAt = await addressOf("anchor", "laptop");
|
|
|
|
// A name the mesh was never told about, under a machine it was — from both machines, because a
|
|
// node must answer for every machine and not only for itself.
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
let got = "";
|
|
for (let i = 0; i < 15 && !got; i++) {
|
|
got = await resolves(machine, "postgres.anchor.internal");
|
|
if (!got) await new Promise((r) => setTimeout(r, 3000));
|
|
}
|
|
assert.equal(got, anchorAt,
|
|
`${machine} does not resolve a service named under anchor: ${got || "(nothing)"}\n\n` +
|
|
`${await diagnose(machine)}`);
|
|
}
|
|
|
|
// Any name at all, which is the whole point: the mesh was never told these exist.
|
|
for (const [name, expected] of [
|
|
["postgres-2.anchor.internal", anchorAt],
|
|
["keycloak.anchor.internal", anchorAt],
|
|
["plex.laptop.internal", laptopAt],
|
|
["radarr.laptop.internal", laptopAt],
|
|
] as const) {
|
|
assert.equal(await resolves("laptop", name), expected,
|
|
`${name} did not resolve to the machine it is named under\n\n${await diagnose("laptop")}`);
|
|
}
|
|
|
|
// The machine's own name still resolves, and to the same place. Two accounts of where a machine
|
|
// is, disagreeing, would be worse than either alone.
|
|
assert.equal(await resolves("laptop", "anchor.internal"), anchorAt);
|
|
|
|
// And what is not the mesh's is not answered by it. The resolver takes over the mesh's names
|
|
// and nothing else, which is what lets a machine keep whatever DNS it already had.
|
|
assert.equal(await resolves("anchor", "something.example.com"), "",
|
|
"the resolver answered for a name that is not the mesh's");
|
|
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
await mesh(`unassign ${machine} resolved-split-dns`);
|
|
await mesh(`unassign ${machine} dnsmasq`);
|
|
}
|
|
await mesh("push");
|
|
});
|
|
|
|
// A real third-party workload, adopted the way the conversion will adopt one.
|
|
//
|
|
// **Everything before this used modules written to exercise the mesh.** This one is software
|
|
// nobody here wrote, taking its credentials the way such software does — from its environment —
|
|
// and needing two containers that reach each other by name. It is the first module that could not
|
|
// have been declared before today: it needs the `network` shape, and it needs a sealed value to
|
|
// reach a container's environment.
|
|
//
|
|
// Its database password is **accepted rather than generated**, which is the whole shape of an
|
|
// adoption: a service that already exists keeps the credential it already has, because minting a
|
|
// new one is how a running application stops being able to reach its own database.
|
|
test("a third-party workload is adopted, with the credential it already had", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
const password = "the-password-it-already-had";
|
|
|
|
await must("anchor", `printf %s ${quote(JSON.stringify({
|
|
module: "adopted-analytics",
|
|
version: "1",
|
|
capabilities: ["container-runtime"],
|
|
"own-secrets": {
|
|
database: "/var/lib/umami/database.env",
|
|
app: "/var/lib/umami/app.env",
|
|
},
|
|
listens: [{ port: 1212, protocol: "tcp", from: "mesh", why: "the analytics page" }],
|
|
resources: [
|
|
{ id: "state", type: "directory", path: "/var/lib/umami", mode: "0700" },
|
|
// The two containers must reach each other by name, which is what this shape is for.
|
|
{ id: "net", type: "network", name: "umami" },
|
|
{
|
|
id: "db", type: "container", name: "umami-db",
|
|
image: pinned("postgres"),
|
|
network: "umami",
|
|
env: { POSTGRES_DB: "umami", POSTGRES_USER: "umami" },
|
|
"env-file": ["/var/lib/umami/database.env"],
|
|
},
|
|
{
|
|
id: "app", type: "container", name: "umami",
|
|
image: pinned("ghcr.io/umami-software/umami"),
|
|
network: "umami",
|
|
env: { DATABASE_TYPE: "postgresql" },
|
|
"env-file": ["/var/lib/umami/app.env"],
|
|
ports: ["1212:3000"],
|
|
},
|
|
],
|
|
}))} > /umami.json`);
|
|
await must("anchor", `docker cp /umami.json mesh-controller:/umami.json`);
|
|
await mesh("module add /umami.json");
|
|
|
|
// **Accepted, not generated.** The value is what the database already answers to; the mesh
|
|
// seals it and cannot read it again. Given whole, as the environment lines the containers read.
|
|
await must("anchor",
|
|
`printf %s ${quote(`POSTGRES_PASSWORD=${password}`)} | ` +
|
|
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics database --from -`);
|
|
await must("anchor",
|
|
`printf %s ${quote(
|
|
`DATABASE_URL=postgresql://umami:${password}@umami-db:5432/umami`)} | ` +
|
|
`docker exec -i mesh-controller /mesh-controller secret accept anchor adopted-analytics app --from -`);
|
|
|
|
await mesh("assign anchor adopted-analytics");
|
|
await mesh("push anchor", 300_000);
|
|
|
|
// Both containers, and the network they share.
|
|
let up = false;
|
|
for (let i = 0; i < 60 && !up; i++) {
|
|
const running = await on("anchor", `docker ps --format '{{.Names}}'`);
|
|
up = running.out.includes("umami-db") && running.out.includes("umami");
|
|
if (!up) await new Promise((r) => setTimeout(r, 5000));
|
|
}
|
|
if (!up) {
|
|
// Everything that could say why, gathered before asserting. "It did not start" is the one
|
|
// thing already known; what is wanted is whether the mesh sent it, whether the host refused
|
|
// it, and what the runtime said when it tried.
|
|
const said = await mesh("status");
|
|
const containers = await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`);
|
|
const applied = await on("anchor",
|
|
`${HOST_PATH} owned 2>&1 | head -30 || echo "the host could not say what it owns"`);
|
|
const files = await on("anchor", `ls -la /var/lib/umami/ 2>&1; ` +
|
|
`for f in /var/lib/umami/*.env; do echo "-- $f"; wc -c "$f"; done 2>&1`);
|
|
const tried = await on("anchor",
|
|
`docker inspect umami-db --format '{{.State.Status}} {{.State.Error}}' 2>&1; ` +
|
|
`docker logs umami-db 2>&1 | tail -15`);
|
|
assert.fail(
|
|
`the workload never started.\n\n` +
|
|
`── what the mesh thinks:\n${said}\n` +
|
|
`── containers:\n${containers.out}\n` +
|
|
`── what the host owns:\n${applied.out}\n` +
|
|
`── what the mesh wrote:\n${files.out}\n` +
|
|
`── the database container:\n${tried.out}\n`);
|
|
}
|
|
|
|
// The environment file the mesh sealed is on the machine and readable only by root.
|
|
const mode = await must("anchor", `stat -c %a /var/lib/umami/database.env`);
|
|
assert.equal(mode.trim(), "600", "a file holding a credential is readable by more than root");
|
|
|
|
// **The assertion that matters: the credential works.** Not that a file arrived — that the
|
|
// database the mesh started answers to the password the mesh was given rather than one it made.
|
|
let connected = { out: "", ok: false };
|
|
for (let i = 0; i < 40 && !connected.ok; i++) {
|
|
connected = await on("anchor",
|
|
`docker exec umami-db psql -U umami -d umami -qAt -c 'select 1'`);
|
|
if (!connected.ok) await new Promise((r) => setTimeout(r, 3000));
|
|
}
|
|
assert.ok(connected.ok, `the database never came up:\n${connected.out}`);
|
|
|
|
const wrong = await on("anchor",
|
|
`docker run --rm --network umami -e PGPASSWORD=not-the-password ${pinned("postgres")} ` +
|
|
`psql -h umami-db -U umami -d umami -qAt -c 'select 1'`);
|
|
assert.ok(!wrong.ok,
|
|
"the database accepted a password nobody gave it, so this proves nothing about the one that was");
|
|
|
|
// And the two containers reach each other by name over the module's own network.
|
|
const reached = await must("anchor",
|
|
`docker run --rm --network umami ${pinned("postgres")} ` +
|
|
`sh -c 'getent hosts umami-db || echo unreachable'`);
|
|
assert.doesNotMatch(reached, /unreachable/,
|
|
"a container could not reach the other by name, so the module's network did nothing");
|
|
});
|
|
|
|
// The real modules, resolved together on one machine.
|
|
//
|
|
// **What this proves without pulling a gigabyte of images**: that five manifests written from the
|
|
// running system resolve as a graph — keycloak's requirement met by postgres's provision,
|
|
// capabilities checked, nothing claiming the same singular thing — and that the declaration the
|
|
// control plane composes is one the host accepts. `plan --json` exists for exactly this: it is
|
|
// the only way to know that what the control plane emits is what the host takes.
|
|
//
|
|
// Running them needs their images stocked and two provisioners built, which is a separate and
|
|
// larger job. This is the half that can be known now, and it is the half where a design fault
|
|
// would live.
|
|
// Three tests lived here that read the mesh's example modules, which moved to the catalogue.
|
|
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood
|
|
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the
|
|
// catalogue's modules together and its gate is the composed declaration accepted and every core
|
|
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates
|
|
// on gitea running but does not yet ask it to answer on its port with the credential it was given,
|
|
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
|
|
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
|
|
// mesh-grant-end-to-end, against the catalogue's redis.
|