jschoubben
35000a236c
A builder that cannot connect sits there, and every outward sign — container up, credential on disk — says it is working. The failure surfaced five minutes later as nothing consuming the build queue, which names no cause at all.
716 lines
38 KiB
TypeScript
716 lines
38 KiB
TypeScript
/**
|
|
* A mesh, raised from nothing, joined by two machines, delivering a credential neither the mesh
|
|
* nor the broker can read.
|
|
*
|
|
* Everything before this proves a part. This proves the parts meet — which is the thing the
|
|
* project keeps saying cannot be checked any other way (novox/hq ADR 0001: every fault of
|
|
* 2026-08-22 was found in production because nothing could be stood up locally).
|
|
*
|
|
* It needs a host binary and the substrate bundle:
|
|
*
|
|
* MESH_LAB_HOST_BINARY=.../mesh-host
|
|
* MESH_LAB_BUNDLE=.../examples/substrate-first-node.lock
|
|
*
|
|
* The bundle's image references are rewritten to the ones this scenario's own registry serves.
|
|
* A digest belongs to whatever registry serves it, so a committed bundle names a registry that is
|
|
* not this one — rewriting is what makes it applicable rather than a placeholder to tidy away.
|
|
*/
|
|
|
|
import { test, before, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { loadScenario } from "../../src/declaration/parse.ts";
|
|
import { raise } from "../../src/lifecycle/raise.ts";
|
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
|
import { labIsUsable, destroyAll } from "./harness.ts";
|
|
import { incus } from "../../src/incus/client.ts";
|
|
import { machineName } from "../../src/lifecycle/names.ts";
|
|
|
|
const capability = await labIsUsable();
|
|
const binary = hostBinaryPath();
|
|
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
|
|
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
|
|
|
|
const skip = !capability.usable
|
|
? `lab not usable: ${capability.why}`
|
|
: !binary || !existsSync(binary)
|
|
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
|
|
: !bundle || !existsSync(bundle)
|
|
? "MESH_LAB_BUNDLE is not set to a substrate bundle (mesh-host examples/)"
|
|
: false;
|
|
|
|
const SCENARIO = "two-nodes";
|
|
let instanceId = "";
|
|
/** The scenario's own registry, which serves the images a module may mirror. */
|
|
let registry = "";
|
|
/** What that registry actually serves, by repository. */
|
|
let stocked: string[] = [];
|
|
|
|
/**
|
|
* The pinned reference for one of the scenario's images.
|
|
*
|
|
* By digest, because the lab's registry drops tags when it stocks: `registry:2` is not there and
|
|
* asking for it fails with "not found", which reads like a missing image rather than a naming
|
|
* convention. A digest is also what a declaration pins, so this is the reference a module would
|
|
* really carry.
|
|
*/
|
|
function pinned(repository: string): string {
|
|
const found = stocked.find((r) => r.slice(r.indexOf("/") + 1, r.indexOf("@")) === repository);
|
|
assert.ok(found, `the scenario stocks no ${repository}; it serves ${stocked.join(", ")}`);
|
|
return found;
|
|
}
|
|
|
|
function quote(s: string): string {
|
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
}
|
|
|
|
async function on(machine: string, command: string, timeoutMs?: number): Promise<{ out: string; ok: boolean }> {
|
|
// Each part on its own line, and stderr redirected once for the whole script.
|
|
//
|
|
// It was `${command} 2>&1; echo ...` on a single line, which quietly broke every command
|
|
// containing a heredoc: the terminator line became `MARKER 2>&1; echo ...`, matched nothing, and
|
|
// the heredoc swallowed the rest of the script — including the echo. `exec 2>&1` needs no
|
|
// trailing text on the command's last line, so a heredoc terminates where it says it does.
|
|
const { stdout } = await exec(instanceId, machine, [
|
|
"sh", "-c", `exec 2>&1\n${command}\necho "__exit=$?"`,
|
|
], timeoutMs);
|
|
const marker = stdout.lastIndexOf("__exit=");
|
|
if (marker < 0) {
|
|
// **Never success.** `Number("")` is 0, so a missing marker used to read as exit 0 — a
|
|
// command whose output was swallowed reported that it worked, which is the one answer a test
|
|
// harness must never give.
|
|
return { out: stdout, ok: false };
|
|
}
|
|
const said = stdout.slice(marker + 7).trim();
|
|
return { out: stdout.slice(0, marker), ok: said === "0" };
|
|
}
|
|
|
|
async function must(machine: string, command: string, timeoutMs?: number): Promise<string> {
|
|
const { out, ok } = await on(machine, command, timeoutMs);
|
|
if (!ok) throw new Error(`${machine}: ${command}\n${out}`);
|
|
return out;
|
|
}
|
|
|
|
/** The control plane, which runs in a container on the first node. */
|
|
async function mesh(command: string, timeoutMs?: number): Promise<string> {
|
|
return must("anchor", `docker exec mesh-control /mesh-control ${command}`, timeoutMs);
|
|
}
|
|
|
|
/**
|
|
* The bundle, with every image reference pointed at this scenario's registry.
|
|
*
|
|
* Matched by repository rather than by the whole reference, because the address and the digest
|
|
* both differ from whatever the committed bundle names — and a bundle that names the wrong
|
|
* registry is not wrong, it is built for a different target.
|
|
*/
|
|
function bundleFor(images: string[]): string {
|
|
let text = readFileSync(bundle, "utf8");
|
|
for (const pinned of images) {
|
|
const repository = pinned.slice(pinned.indexOf("/") + 1, pinned.indexOf("@"));
|
|
const escaped = repository.replaceAll("/", "\\/").replaceAll(".", "\\.");
|
|
text = text.replaceAll(
|
|
new RegExp(`[A-Za-z0-9_.:-]+\\/${escaped}@sha256:[0-9a-f]+`, "g"),
|
|
pinned,
|
|
);
|
|
}
|
|
return text;
|
|
}
|
|
|
|
/** Take a token out of what `token issue` printed. It is the one base64url blob on its own line. */
|
|
function tokenFrom(said: string): string {
|
|
const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" "));
|
|
assert.ok(found, `no token in:\n${said}`);
|
|
return found;
|
|
}
|
|
|
|
before(async () => {
|
|
if (skip) return;
|
|
const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), {});
|
|
instanceId = raised.instanceId;
|
|
|
|
// The first node raises everything from a file rather than from a bundle built into the binary,
|
|
// because the digests are this registry's and are not known until it is up.
|
|
await must("anchor", `cat > /tmp/substrate.lock <<'MESHBUNDLE'\n${bundleFor(raised.images)}\nMESHBUNDLE`);
|
|
await must("anchor", `${HOST_PATH} apply /tmp/substrate.lock`);
|
|
stocked = raised.images;
|
|
const first = raised.images[0];
|
|
assert.ok(first, "the scenario stocked no images, so nothing can be mirrored");
|
|
registry = first.slice(0, first.indexOf("/"));
|
|
|
|
// A build machine, so anything here can ask the mesh to build something. Placed rather than
|
|
// assumed: nothing else in this scenario would start one.
|
|
if (builder) {
|
|
await incus([
|
|
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
|
|
"--mode", "0755",
|
|
], 180_000);
|
|
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
|
|
await must("anchor",
|
|
`MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
|
|
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
|
|
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
|
|
}
|
|
}, { timeout: 1_800_000 });
|
|
|
|
after(async () => {
|
|
if (instanceId) await destroy(instanceId);
|
|
await destroyAll(`${SCENARIO}-`);
|
|
}, { timeout: 600_000 });
|
|
|
|
test("a bare machine becomes a mesh", { skip, timeout: 600_000 }, async () => {
|
|
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) {
|
|
assert.match(running, new RegExp(container), `${container} is not running`);
|
|
}
|
|
// Answering, not merely up. A container that is running is not a control plane that replies —
|
|
// a distinction this project has already paid for once.
|
|
assert.ok((await mesh("status")).length > 0);
|
|
});
|
|
|
|
test("both machines join it, and the token is all they need", { skip, timeout: 900_000 }, async () => {
|
|
for (const [machine, node] of [["anchor", "anchor"], ["laptop", "laptop"]] as const) {
|
|
await mesh(`node add ${node}`);
|
|
const token = tokenFrom(await mesh(`token issue --node ${node}`));
|
|
// No --name. The token says what the mesh calls the machine, which is the fault this walk
|
|
// found the first time it was run.
|
|
const said = await must(machine, `${HOST_PATH} enrol --token ${quote(token)}`);
|
|
assert.match(said, new RegExp(`enrolled as ${node}`), said);
|
|
assert.match(said, /sealing key/, "no sealing key was generated");
|
|
}
|
|
|
|
const recorded = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select name from node where sealing_key is not null order by name"`);
|
|
assert.equal(recorded.trim().split("\n").map((l) => l.trim()).sort().join(","), "anchor,laptop",
|
|
"the mesh did not record a sealing key for both machines");
|
|
});
|
|
|
|
test("a credential reaches both ends and the mesh holds neither", { skip, timeout: 900_000 }, async () => {
|
|
// The whole argument, on real machines: the two ends must hold the SAME password, and it must
|
|
// appear nowhere the mesh or the broker could read it.
|
|
await must("anchor", `printf %s '{"module":"postgres","version":"1",` +
|
|
`"provides":[{"name":"database","scope":"mesh"}],"serves":{"database":{"port":5432}},` +
|
|
`"grants":{"database":"/var/lib/mesh-host/grants"},` +
|
|
`"receives":{"database":"/var/lib/mesh-host/grants/mesh.json"},"resources":[]}' > /tmp/pg.json`);
|
|
await must("anchor", `printf %s '{"module":"meshboard","version":"1",` +
|
|
`"requires":["database"],"contributes":{"database":{"name":"meshboard"}},` +
|
|
`"binds":{"database":"/etc/meshboard/database.json"},` +
|
|
`"secrets":{"database":"/etc/meshboard/database.password"},"resources":[]}' > /tmp/app.json`);
|
|
await must("anchor", `docker cp /tmp/pg.json mesh-control:/pg.json`);
|
|
await must("anchor", `docker cp /tmp/app.json mesh-control:/app.json`);
|
|
await mesh("module add /pg.json");
|
|
await mesh("module add /app.json");
|
|
|
|
await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
|
|
await mesh("overlay place laptop --site lab");
|
|
for (const node of ["anchor", "laptop"]) await mesh(`assign ${node} networking`);
|
|
await mesh("assign anchor postgres");
|
|
await mesh("assign laptop meshboard");
|
|
|
|
for (const machine of ["anchor", "laptop"]) {
|
|
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
|
|
}
|
|
await mesh("push");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
|
|
const onProvider = (await must("anchor", `cat /var/lib/mesh-host/grants/laptop.secret`)).trim();
|
|
assert.ok(onConsumer.length >= 40, `the consumer's credential is ${onConsumer.length} characters`);
|
|
assert.equal(onConsumer, onProvider,
|
|
"the two ends hold different passwords, so nothing could ever authenticate");
|
|
|
|
// Only the machine it is for may read it.
|
|
assert.match(await must("laptop", `stat -c %a /etc/meshboard/database.password`), /^600/);
|
|
|
|
// And it is nowhere it could have been read on the way. The declaration crossed the broker; the
|
|
// database is the control plane's; the state is what the node reported back.
|
|
for (const [machine, where] of [
|
|
["laptop", "/var/lib/mesh-host/declared.json"],
|
|
["laptop", "/var/lib/mesh-host/state.json"],
|
|
["anchor", "/var/lib/mesh-host/declared.json"],
|
|
] as const) {
|
|
const { out } = await on(machine, `grep -c ${quote(onConsumer)} ${where}`);
|
|
assert.equal(out.trim(), "0", `the password is in ${where} on ${machine}`);
|
|
}
|
|
const inTheMesh = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select count(*) from secret where for_consumer like '%${onConsumer}%' ` +
|
|
`or for_provider like '%${onConsumer}%'"`);
|
|
assert.equal(inTheMesh.trim(), "0", "the control plane's database holds the password in the clear");
|
|
});
|
|
|
|
test("the consumer is also told where its database is", { skip, timeout: 300_000 }, async () => {
|
|
// A password with no address is not a connection. This is the readable half, which stays
|
|
// readable on purpose — it is the secret half that could not be composed, not this one.
|
|
const told = JSON.parse(await must("laptop", `cat /etc/meshboard/database.json`));
|
|
assert.equal(told.from, "anchor");
|
|
assert.equal(told.at, "anchor.internal");
|
|
assert.equal(told.serves.port, 5432);
|
|
// And the name it resolves to was written by the mesh as well, on this machine.
|
|
assert.match(await must("laptop", `grep anchor.internal /etc/hosts`), /10\.42\.0\.\d+/);
|
|
});
|
|
|
|
test("when a machine cannot do what it was told, the mesh says which and why", { skip, timeout: 900_000 }, async () => {
|
|
// Demonstrated with inserted rows first, which proves the query and not the path. This sends a
|
|
// real machine something it will genuinely fail at, and asks the mesh afterwards.
|
|
//
|
|
// A package that does not exist, because that is a failure of the ordinary kind: the host tries,
|
|
// the package manager says no, and some of the declaration is applied and some is not — which
|
|
// is the situation `status` exists to distinguish from a machine that refused everything.
|
|
await must("anchor", `printf %s '{"module":"impossible","version":"1","resources":[` +
|
|
`{"id":"nothing","type":"package","package":"a-package-that-does-not-exist"}]}' > /tmp/imp.json`);
|
|
await must("anchor", `docker cp /tmp/imp.json mesh-control:/imp.json`);
|
|
await mesh("module add /imp.json");
|
|
await mesh("assign laptop impossible");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 10_000));
|
|
|
|
const said = await mesh("status");
|
|
assert.match(said, /not doing what they were told/, said);
|
|
assert.match(said, /laptop/, said);
|
|
// The machine's own words about the resource that failed, not a summary written at this end.
|
|
assert.match(said, /impossible\.nothing/, `the failing resource is not named:\n${said}`);
|
|
|
|
// And the distinction survives: this machine FAILED, it did not refuse. Refused means it is
|
|
// exactly as it was; failed means it is in a state nobody declared, and they are fixed in
|
|
// different places.
|
|
assert.match(said, /laptop\s+failed/, said);
|
|
|
|
// The other machine is not implicated.
|
|
assert.doesNotMatch(said.split("not heard from")[0] ?? said, /anchor\s+(failed|refused)/,
|
|
"a machine that did as it was told is listed as wrong");
|
|
});
|
|
|
|
test("a declaration waits for a machine that is switched off", { skip, timeout: 900_000 }, async () => {
|
|
// A machine is disconnected as an ordinary situation, not an exception (novox/hq ADR 0004), so
|
|
// a push to one that is not listening must wait rather than vanish. The queue is durable and the
|
|
// message persistent, which ought to be enough — but a lost declaration is silent, and "ought to
|
|
// be" is not a property.
|
|
//
|
|
// The machine is not merely idle here: its host is stopped, so nothing is consuming its queue.
|
|
|
|
// Nothing this test asserts should depend on what another left behind. The machine still has a
|
|
// deliberately-impossible module from the test above, and while that is assigned the mesh never
|
|
// updates its account of what the machine holds — a partial report is not an account, on
|
|
// purpose (novox/hq 04-ISSUES/010).
|
|
await mesh("unassign laptop impossible");
|
|
|
|
// Stop listening, and prove it stopped — a test that pushed to a machine that was still running
|
|
// would pass having checked nothing.
|
|
//
|
|
// By process name, never by matching the command line: `pkill -f` matches the shell running it
|
|
// too, which kills the connection carrying the command and hangs the caller waiting for a reply
|
|
// that will never come. Cost an hour once, in this file.
|
|
await must("laptop", `pkill -x mesh-host || true; sleep 1`);
|
|
const listening = await on("laptop", `pgrep -x mesh-host`);
|
|
assert.equal(listening.ok, false, "the host is still running, so this proves nothing");
|
|
|
|
await must("anchor", `printf %s '{"module":"while-away","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-while-away","content":"waited"}]}' > /tmp/away.json`);
|
|
await must("anchor", `docker cp /tmp/away.json mesh-control:/away.json`);
|
|
await mesh("module add /away.json");
|
|
await mesh("assign laptop while-away");
|
|
await mesh("push laptop");
|
|
|
|
// Nothing has happened on the machine, because nothing is there to do it.
|
|
const before = await on("laptop", `test -f /etc/mesh-while-away`);
|
|
assert.equal(before.ok, false, "a machine with no host applied a declaration");
|
|
|
|
// And now it listens again. No second push, and nobody says anything.
|
|
await must("laptop", `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 8`);
|
|
let arrived = false;
|
|
for (let i = 0; i < 20 && !arrived; i++) {
|
|
arrived = (await on("laptop", `test -f /etc/mesh-while-away`)).ok;
|
|
if (!arrived) await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
if (!arrived) {
|
|
// Everything needed to tell "the message was never queued" from "the host never read it".
|
|
const log = await on("laptop", `tail -20 /var/log/mesh-host.log`);
|
|
const queues = await on("anchor",
|
|
`docker exec mesh-broker lavinmqctl list_queues name messages 2>&1 | head -10`);
|
|
const owned = await on("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt -c "select name, outcome from node_report r join node n on n.id=r.node"`);
|
|
assert.fail(`a declaration sent to a switched-off machine was lost\n` +
|
|
`--- the host's log ---\n${log.out}\n--- the broker's queues ---\n${queues.out}\n` +
|
|
`--- what each machine last did ---\n${owned.out}`);
|
|
}
|
|
assert.equal((await must("laptop", `cat /etc/mesh-while-away`)).trim(), "waited");
|
|
|
|
// And the mesh's account of what that machine holds catches up too, or a later declaration
|
|
// would tell it to remove what it has just been given.
|
|
await new Promise((r) => setTimeout(r, 4000));
|
|
const owned = await must("anchor",
|
|
`docker exec mesh-store psql -U postgres -d inventory -qAt ` +
|
|
`-c "select owned from node where name = 'laptop'"`);
|
|
assert.match(owned, /while-away\.note/, `the mesh does not know the machine holds it: ${owned}`);
|
|
});
|
|
|
|
test("unassigning takes away exactly what it should", { skip, timeout: 900_000 }, async () => {
|
|
// Removal is the half nobody tests. The mesh takes away what IT declared and no longer declares,
|
|
// and never what the machine raised for itself from its bundle — which is the fault that
|
|
// destroyed a substrate once (novox/hq 04-ISSUES/010).
|
|
//
|
|
// Two modules, so the test can tell "removed the right one" from "removed everything".
|
|
for (const [name, path] of [["kept", "/etc/mesh-kept"], ["going", "/etc/mesh-going"]] as const) {
|
|
await must("anchor", `printf %s '{"module":"${name}","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"${path}","content":"${name}"}]}' > /tmp/${name}.json`);
|
|
await must("anchor", `docker cp /tmp/${name}.json mesh-control:/${name}.json`);
|
|
await mesh(`module add /${name}.json`);
|
|
await mesh(`assign anchor ${name}`);
|
|
}
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok, "the first module did not arrive");
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-going`)).ok, "the second module did not arrive");
|
|
|
|
await mesh("unassign anchor going");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
|
|
assert.equal((await on("anchor", `test -f /etc/mesh-going`)).ok, false,
|
|
"an unassigned module's file is still there");
|
|
assert.ok((await on("anchor", `test -f /etc/mesh-kept`)).ok,
|
|
"unassigning one module took another one's file with it");
|
|
|
|
// And the substrate this machine raised from its own bundle is untouched. It was not declared by
|
|
// the mesh, so the mesh must never remove it — the machine would take its own control plane
|
|
// away, which is exactly what happened before origins existed.
|
|
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
for (const container of ["mesh-store", "mesh-broker", "mesh-control"]) {
|
|
assert.match(running, new RegExp(container),
|
|
`${container} was removed by a declaration that never declared it`);
|
|
}
|
|
});
|
|
|
|
test("a machine keeps what it was given when the mesh says nothing about it", { skip, timeout: 600_000 }, async () => {
|
|
// The other direction of the same rule. A node that is sent a declaration mentioning none of its
|
|
// private network must not lose it: the network came from a module that is still assigned, and
|
|
// "not in this message" is not "no longer wanted".
|
|
assert.ok((await on("laptop", `test -f /etc/wireguard/mesh0.conf`)).ok,
|
|
"the private network's configuration is gone");
|
|
assert.ok((await on("laptop", `grep -q anchor.internal /etc/hosts`)).ok,
|
|
"the mesh's names are gone");
|
|
});
|
|
|
|
test("a machine that fell behind catches up without being named", { skip, timeout: 900_000 }, async () => {
|
|
// `status` says which machines are not doing what they were told; something has to act on it.
|
|
// `push --behind` is that something, and it is a command rather than a timer to begin with —
|
|
// a scheduler is then a scheduler over this, rather than a second path to the same act.
|
|
|
|
// Break one machine, in a way that is fixable: a package that does not exist yet.
|
|
await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
|
|
`{"id":"pkg","type":"package","package":"a-package-that-does-not-exist-yet"},` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
|
|
await must("anchor", `docker cp /tmp/fix.json mesh-control:/fix.json`);
|
|
await mesh("module add /fix.json");
|
|
await mesh("assign laptop fixable");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
assert.match(await mesh("status"), /laptop\s+failed/, "the machine did not report a failure");
|
|
// And the resources that COULD be applied were — one broken thing no longer blocks the rest
|
|
// (novox/hq 04-ISSUES/011).
|
|
assert.ok((await on("laptop", `test -f /etc/mesh-fixable`)).ok,
|
|
"a resource after the failing one was never attempted");
|
|
|
|
// Nothing is behind on the other machine, so nothing is pushed to it.
|
|
const named = await mesh("push --behind");
|
|
assert.match(named, /laptop/, named);
|
|
assert.doesNotMatch(named, /sent anchor/, `a machine that was fine was pushed to:\n${named}`);
|
|
|
|
// Fix the cause, the way somebody would: the module stops asking for the impossible thing.
|
|
await must("anchor", `printf %s '{"module":"fixable","version":"1","resources":[` +
|
|
`{"id":"note","type":"file","path":"/etc/mesh-fixable","content":"here"}]}' > /tmp/fix.json`);
|
|
await must("anchor", `docker cp /tmp/fix.json mesh-control:/fix.json`);
|
|
await mesh("module add /fix.json");
|
|
|
|
// And nobody names the machine.
|
|
await mesh("push --behind");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
const after = await mesh("status");
|
|
assert.doesNotMatch(after, /laptop\s+(failed|refused)/,
|
|
`the machine did not recover:\n${after}`);
|
|
|
|
// With nothing behind, it says so rather than doing nothing quietly.
|
|
assert.match(await mesh("push --behind"), /every machine is doing what it was told/);
|
|
});
|
|
|
|
test("the mesh runs its own artifact store", {
|
|
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
|
|
timeout: 900_000,
|
|
}, async () => {
|
|
// Artifacts go to a registry, and the only registries that existed were raised by the lab or by
|
|
// the bootstrap bundle. A mesh had no way to run its own.
|
|
//
|
|
// Chicken and egg, resolved the way the bootstrap's is: the scenario's registry serves the image
|
|
// the module mirrors, and the module then runs a registry of the mesh's own.
|
|
await must("anchor", `mkdir -p /root/registry && printf %s '{"module":"registry","version":"1",` +
|
|
`"provides":[{"name":"artifact-store","scope":"mesh"}],` +
|
|
`"capabilities":["container-runtime"],` +
|
|
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
|
|
`"serves":{"artifact-store":{"port":5000}},` +
|
|
`"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"${pinned("registry")}"}]},` +
|
|
`"resources":[` +
|
|
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
|
|
`{"id":"store","type":"container","name":"mesh-registry","artifact":"registry",` +
|
|
`"ports":["5000:5000"],"volumes":["mesh-registry-data:/var/lib/registry"]}]}' ` +
|
|
`> /root/registry/module.json`);
|
|
await must("anchor", `cd /root/registry && git init -q . && git add -A && ` +
|
|
`git -c user.email=lab -c user.name=lab commit -qm registry`);
|
|
|
|
await mesh("build /root/registry --wait 300s", 420_000);
|
|
await mesh("assign anchor registry");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 12_000));
|
|
|
|
// Running, and answering — a container that is up is not a registry that replies.
|
|
assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
|
|
let answers = false;
|
|
for (let i = 0; i < 20 && !answers; i++) {
|
|
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
|
|
if (!answers) await new Promise((r) => setTimeout(r, 2000));
|
|
}
|
|
assert.ok(answers, "the mesh's own registry is running and does not answer");
|
|
|
|
// And reachable from another machine over the private network, which is the whole point of an
|
|
// artifact store being a mesh-scoped provision.
|
|
assert.ok((await on("laptop", `curl -sf http://anchor.internal:5000/v2/ -o /dev/null`)).ok,
|
|
"the artifact store is not reachable from another machine, so nothing else can use it");
|
|
});
|
|
|
|
test("a machine serves its internal name with a certificate the mesh issued", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
|
|
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
|
|
// moment something connects, which is the worst place to find out.
|
|
await must("anchor", `printf %s '{"module":"served","version":"1",` +
|
|
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
|
|
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
|
|
`> /tmp/served.json`);
|
|
await must("anchor", `docker cp /tmp/served.json mesh-control:/served.json`);
|
|
await mesh("module add /served.json");
|
|
await mesh("assign anchor served");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 8000));
|
|
|
|
assert.ok((await on("anchor", `test -s /etc/mesh/serving.crt`)).ok, "no certificate arrived");
|
|
assert.ok((await on("anchor", `test -s /etc/mesh/authority.crt`)).ok, "no authority arrived");
|
|
|
|
// The name it was issued for is the one the mesh gave this machine.
|
|
const named = await must("anchor",
|
|
`openssl x509 -in /etc/mesh/serving.crt -noout -ext subjectAltName 2>/dev/null || ` +
|
|
`docker run --rm -v /etc/mesh:/m ${pinned("registry")} sh -c ` +
|
|
`"apk add --no-cache openssl >/dev/null 2>&1; openssl x509 -in /m/serving.crt -noout -text" | grep -A1 'Alternative'`);
|
|
assert.match(named, /anchor\.internal/, `the certificate is not for this machine's name:\n${named}`);
|
|
|
|
// And a real handshake: the machine serves TLS with the key it generated, and another machine
|
|
// verifies it against the mesh's authority and nothing else.
|
|
await must("anchor", `openssl s_server -cert /etc/mesh/serving.crt ` +
|
|
`-key /var/lib/mesh-host/serving.key -accept 8443 -naccept 1 -quiet ` +
|
|
`> /var/log/tls.log 2>&1 & sleep 2`);
|
|
await must("laptop", `mkdir -p /etc/mesh`);
|
|
const authority = await must("anchor", `cat /etc/mesh/authority.crt`);
|
|
await must("laptop", `cat > /etc/mesh/authority.crt <<'MESHCA'\n${authority}\nMESHCA`);
|
|
|
|
const shook = await on("laptop",
|
|
`echo | openssl s_client -connect anchor.internal:8443 ` +
|
|
`-CAfile /etc/mesh/authority.crt -verify_return_error -brief 2>&1`);
|
|
assert.ok(shook.ok, `the handshake failed:\n${shook.out}\n` +
|
|
`what the server said:\n${(await on("anchor", `cat /var/log/tls.log`)).out}`);
|
|
assert.match(shook.out, /Verification: OK/, shook.out);
|
|
});
|
|
|
|
test("a machine filters exactly what its modules declared, and nothing else", {
|
|
skip, timeout: 900_000,
|
|
}, async () => {
|
|
// The rule set is derived from what is assigned, not kept in step by hand — and the proof that
|
|
// matters is not that a file arrived but that packets are treated differently because of it.
|
|
// A rule nothing enforces is the fault this mechanism exists to remove (novox/hq 04-ISSUES/003).
|
|
//
|
|
// Note what the module cannot contain: an action. The link may not carry one (novox/hq ADR 0005),
|
|
// so the mesh writes the rule set and declares that a service must reflect it. `restart-on` is
|
|
// the shape that rule leaves, and this is the first thing to use it for its real purpose.
|
|
// A listener is written to a file rather than squeezed through three levels of shell quoting.
|
|
// The first attempt did the latter, never started, and the test failed on its own setup —
|
|
// which reads exactly like the firewall working.
|
|
await must("laptop", `cat > /root/listen.py <<'LISTENER'\n` +
|
|
`import socket, sys, threading\n` +
|
|
`def serve(port):\n` +
|
|
` s = socket.socket()\n` +
|
|
` s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n` +
|
|
` s.bind(("0.0.0.0", port))\n` +
|
|
` s.listen(8)\n` +
|
|
` while True:\n` +
|
|
` c, _ = s.accept()\n` +
|
|
` c.send(str(port).encode())\n` +
|
|
` c.close()\n` +
|
|
`for p in (9101, 9102):\n` +
|
|
` threading.Thread(target=serve, args=(p,), daemon=True).start()\n` +
|
|
`threading.Event().wait()\n` +
|
|
`LISTENER`);
|
|
await must("laptop", `nohup python3 /root/listen.py > /var/log/listen.log 2>&1 & sleep 2`);
|
|
|
|
const reach = async (port: number) => {
|
|
const said = await on("anchor",
|
|
`timeout 5 python3 -c "import socket;s=socket.create_connection(('192.0.2.20',${port}),4);` +
|
|
`print(s.recv(32).decode());s.close()"`);
|
|
return said.ok;
|
|
};
|
|
|
|
// Reachable before any rule set exists, so what changes afterwards is the rule set and not the
|
|
// listener. Without this the test would pass against a service that never started.
|
|
assert.ok(await reach(9101), "the declared port never opened, so nothing below tests anything");
|
|
assert.ok(await reach(9102), "the undeclared port never opened");
|
|
|
|
await must("anchor", `printf %s '{"module":"talker","version":"1",` +
|
|
`"listens":[{"port":9101,"from":"mesh","why":"the thing this test is about"}],` +
|
|
`"resources":[]}' > /tmp/talker.json`);
|
|
// The rule set goes where this machine's nftables unit reads from, and the unit is declared to
|
|
// reflect it. No command anywhere.
|
|
await must("anchor", `printf %s '{"module":"firewall","version":"1",` +
|
|
`"capabilities":["firewall"],` +
|
|
`"filtering":{"into":"/etc/nftables.conf"},` +
|
|
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
|
|
`{"id":"filter","type":"service","unit":"nftables.service","state":"running",` +
|
|
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/firewall.json`);
|
|
for (const f of ["talker", "firewall"]) {
|
|
await must("anchor", `docker cp /tmp/${f}.json mesh-control:/${f}.json`);
|
|
await mesh(`module add /${f}.json`);
|
|
}
|
|
await mesh("assign laptop talker");
|
|
await mesh("assign laptop firewall");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 20_000));
|
|
|
|
const written = await must("laptop", `cat /etc/nftables.conf`);
|
|
// A rule names its source. Not decoration: it is the only thing that answers "why is this open".
|
|
assert.match(written, /# talker . the thing this test is about/,
|
|
`the rule does not name what caused it:\n${written}`);
|
|
assert.match(written, /192\.0\.2\.\d+/, `"from the mesh" resolved to nothing:\n${written}`);
|
|
assert.doesNotMatch(written, /dport 9102/, `a port no module declared was opened:\n${written}`);
|
|
|
|
// Loaded, not merely written. The service was restarted because a file it reflects changed.
|
|
const table = await must("laptop", `nft list table inet mesh`);
|
|
assert.match(table, /dport 9101 accept/, `the rule set was never loaded:\n${table}`);
|
|
|
|
// And it filters. The declared port answers from another machine; the undeclared one does not.
|
|
assert.ok(await reach(9101),
|
|
"the declared port is closed, so the machine is filtering more than it was told to");
|
|
assert.ok(!(await reach(9102)),
|
|
"a port no module declared is still reachable, so the rule set restricts nothing");
|
|
|
|
// The machine did not lock itself out of the mesh: it is still taking declarations.
|
|
assert.doesNotMatch(await mesh("status"), /laptop\s+(failed|refused)/,
|
|
"the machine stopped doing what it was told after applying its own rule set");
|
|
|
|
// Removing the module that wanted the port closes it, with nobody editing a rule. This is the
|
|
// whole claim of a derived firewall, and it is also the second load — which must replace the
|
|
// table rather than add to it.
|
|
await mesh("unassign laptop talker");
|
|
await mesh("push laptop");
|
|
await new Promise((r) => setTimeout(r, 20_000));
|
|
assert.ok(!(await reach(9101)),
|
|
"the port stayed open after the module that wanted it was removed");
|
|
});
|
|
|
|
test("the builder is a module the mesh assigns, with a credential the mesh delivered", {
|
|
skip: skip || (!builder ? "set MESH_LAB_BUILDER to a built mesh-builder" : false),
|
|
timeout: 900_000,
|
|
}, async () => {
|
|
// Until this, the builder was a program somebody started on a machine with whatever credential
|
|
// they had to hand — in practice the broker's administrative one. A program documented as
|
|
// holding its own credential and given somebody else's is worse than one with no story at all.
|
|
//
|
|
// So: the mesh issues a scoped account, seals it to the machine, and delivers it with the
|
|
// declaration. Nobody types it and the mesh cannot read it back.
|
|
await must("anchor", `mkdir -p /root/builder && printf %s '{"module":"builder","version":"1",` +
|
|
`"requires":["artifact-store"],"capabilities":["container-runtime"],` +
|
|
`"claims":[{"name":"the-build-machine","scope":"node"}],` +
|
|
`"binds":{"artifact-store":"/var/lib/mesh/builder/artifact-store.json"},` +
|
|
`"needs":{"broker":"/var/lib/mesh/builder/broker"},` +
|
|
`"build":{"artifacts":[{"name":"builder","kind":"upstream",` +
|
|
`"from":"${pinned("mesh-builder")}"}]},` +
|
|
`"resources":[` +
|
|
`{"id":"state","type":"directory","path":"/var/lib/mesh/builder","mode":"0700"},` +
|
|
`{"id":"workspace","type":"directory","path":"/var/lib/mesh/builder/workspace","mode":"0700"},` +
|
|
`{"id":"run","type":"container","name":"mesh-builder","artifact":"builder",` +
|
|
`"network":"host",` +
|
|
`"volumes":["/var/lib/mesh/builder:/var/lib/mesh/builder",` +
|
|
`"/var/run/docker.sock:/var/run/docker.sock"],` +
|
|
`"env":{"MESH_BROKER_FILE":"/var/lib/mesh/builder/broker",` +
|
|
`"MESH_BINDING":"/var/lib/mesh/builder/artifact-store.json",` +
|
|
`"MESH_WORKSPACE":"/var/lib/mesh/builder/workspace"}}]}' > /root/builder/module.json`);
|
|
await must("anchor", `cd /root/builder && git init -q . && git add -A && ` +
|
|
`git -c user.email=lab -c user.name=lab commit -qm builder`);
|
|
|
|
// The builder's own image is built by the builder that is already running — the same
|
|
// chicken-and-egg as the registry, resolved the same way. The one started by hand does this
|
|
// last piece of work and is then replaced by the module it just built.
|
|
await mesh("build /root/builder --wait 300s", 420_000);
|
|
|
|
// The mesh makes the account and seals the URL to this machine. Nothing is printed that would
|
|
// work if it were pasted somewhere else.
|
|
const issued = await mesh("builder issue lab-builder --node anchor");
|
|
assert.match(issued, /sealed to anchor/, issued);
|
|
assert.doesNotMatch(issued, /amqps:\/\/lab-builder:/,
|
|
"the credential was printed, so the one copy that matters is on a terminal");
|
|
|
|
// Now the hand-started one goes, or two builders race for the same queue and whichever answers
|
|
// proves nothing. By process name: `pkill -f` matches the shell running it too, which kills the
|
|
// connection carrying the command and hangs the caller waiting for a reply that will never
|
|
// come. Cost an hour once, in this file.
|
|
await on("anchor", `pkill -x mesh-builder`);
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
assert.ok(!(await on("anchor", `pgrep -x mesh-builder`)).ok,
|
|
"the hand-started builder is still running, so this would test that one");
|
|
|
|
await mesh("assign anchor builder");
|
|
await mesh("push anchor");
|
|
await new Promise((r) => setTimeout(r, 20_000));
|
|
|
|
const running = await must("anchor", `docker ps --format '{{.Names}}'`);
|
|
assert.match(running, /mesh-builder/,
|
|
`the builder was assigned and is not running:\n${running}\n` +
|
|
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
|
|
|
|
// Running is not connected. A builder that cannot reach the broker sits there, and every
|
|
// outward sign — the container is up, the credential is on disk — says it is working.
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
const said = await on("anchor", `docker logs mesh-builder 2>&1 | tail -20`);
|
|
assert.doesNotMatch(said.out, /cannot reach the broker/,
|
|
`the builder is running and cannot reach the broker:\n${said.out}`);
|
|
|
|
// The credential arrived, is readable only by the machine, and is the scoped account rather
|
|
// than the broker's own.
|
|
assert.match(await must("anchor", `stat -c %a /var/lib/mesh/builder/broker`), /^600/);
|
|
const credential = await must("anchor", `cat /var/lib/mesh/builder/broker`);
|
|
assert.match(credential, /"url":"amqps:\/\/lab-builder:/,
|
|
"the builder is using an account that is not its own");
|
|
assert.doesNotMatch(credential, /guest:guest/, "the builder holds the broker's own account");
|
|
// And what to check the broker against. A mesh's broker presents a certificate of the mesh's
|
|
// own, so a URL alone reaches only a broker some public authority vouches for — which is no
|
|
// mesh broker at all, and fails at TLS with an error about an unknown authority.
|
|
assert.match(credential, /"fingerprint":"[0-9a-f]{64}"/,
|
|
`the builder was given nothing to verify the broker with:\n${credential}`);
|
|
|
|
// And it works: the mesh asks this builder to build something, and it does. Answering is the
|
|
// only proof that the delivered credential authenticates — a container that is up with a
|
|
// credential it cannot use looks identical from outside.
|
|
await must("anchor", `mkdir -p /root/built && printf %s '{"module":"built","version":"1",` +
|
|
`"resources":[{"id":"marker","type":"file","path":"/etc/built","content":"yes","mode":"0644"}]}' ` +
|
|
`> /root/built/module.json`);
|
|
await must("anchor", `cd /root/built && git init -q . && git add -A && ` +
|
|
`git -c user.email=lab -c user.name=lab commit -qm built`);
|
|
await mesh("build /root/built --wait 300s", 420_000);
|
|
|
|
// Naming the module, and not merely containing its name: `builds` says "nothing has been built
|
|
// yet" when there is nothing, and that sentence contains the word this was matching on.
|
|
const recorded = await mesh("builds built");
|
|
assert.doesNotMatch(recorded, /nothing has been built/,
|
|
`the build was accepted and no build was recorded against the module:\n${recorded}`);
|
|
assert.match(recorded, /built/, recorded);
|
|
});
|