events: an e2e test — an emitted event reaches the audit trail over the mesh's broker #2
+40
@@ -35,6 +35,14 @@ const USAGE = `mesh-lab — raise a disposable mesh on one machine
|
||||
suite [paths...] [--no-build] rebuild the artifacts, run the end-to-end tests, leave a receipt
|
||||
last-run whether the last run still counts; non-zero when it does not
|
||||
|
||||
connect [instance] reach the standing scenario from this workstation, by name
|
||||
disconnect give the address back and stop answering those names
|
||||
connected what is reachable right now
|
||||
|
||||
A scenario is a closed address space, so only one can be reachable at a time: connect refuses
|
||||
rather than guessing which you meant. It needs root for an address and a resolver rule, and
|
||||
disconnect puts both back.
|
||||
|
||||
Set MESH_LAB_INCUS if the daemon needs a different invocation, e.g. "sudo -n incus".
|
||||
`;
|
||||
|
||||
@@ -182,6 +190,38 @@ async function main(): Promise<void> {
|
||||
return;
|
||||
}
|
||||
|
||||
case "connect": {
|
||||
const { connect } = await import("./lifecycle/connect.ts");
|
||||
const reached = await connect(rest[0]);
|
||||
console.log(`connected to ${reached.instanceId} as ${reached.address} on ${reached.bridge}\n`);
|
||||
console.log("these answer here now:");
|
||||
for (const [machine, address] of Object.entries(reached.machines).sort()) {
|
||||
console.log(` anything.${machine}.internal → ${address}`);
|
||||
}
|
||||
console.log(`\ntry: curl -sI http://${Object.keys(reached.machines)[0]}.internal`);
|
||||
console.log("run `mesh-lab disconnect` when finished — these names are only true while");
|
||||
console.log("that scenario is standing.");
|
||||
return;
|
||||
}
|
||||
|
||||
case "disconnect": {
|
||||
const { disconnect } = await import("./lifecycle/connect.ts");
|
||||
for (const line of await disconnect()) console.log(line);
|
||||
return;
|
||||
}
|
||||
|
||||
case "connected": {
|
||||
const { connection } = await import("./lifecycle/connect.ts");
|
||||
const now = await connection();
|
||||
if (now.length === 0) {
|
||||
console.log("nothing is connected");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
for (const line of now) console.log(` ${line}`);
|
||||
return;
|
||||
}
|
||||
|
||||
case "raise": {
|
||||
const path = rest[0] ?? fail("raise needs a scenario file");
|
||||
const scenario = loadScenario(path);
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* Letting the workstation reach one scenario by name, on purpose and temporarily.
|
||||
*
|
||||
* **A scenario is a closed address space** ([ADR 0016](../../02-DECISIONS/0016-the-lab.md)): two
|
||||
* raised from the same declaration hold the same addresses and never meet, because nothing joins
|
||||
* their links. That is what lets two identical scenarios run at once, and it is why the lab talks
|
||||
* to machines through the hypervisor's own channel rather than over IP.
|
||||
*
|
||||
* Reaching in from the workstation breaks that, so it is **opt-in, one scenario at a time, and
|
||||
* reversible**. It refuses when more than one is standing rather than guessing which was meant —
|
||||
* the failure it exists to avoid is not an error but one scenario's traffic arriving in another.
|
||||
*
|
||||
* **Names resolve to the segment address, not the overlay one.** Inside the mesh a name answers
|
||||
* with a machine's private-network address; from here that would need the workstation on the
|
||||
* overlay, which is a much larger door to open. The segment address reaches the same machine and
|
||||
* the same ports, which is what somebody opening a board in a browser actually needs.
|
||||
*/
|
||||
|
||||
import { writeFileSync, unlinkSync, existsSync, readFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
import { incus, incusOk, taggedInstances, taggedNetworks } from "../incus/client.ts";
|
||||
import { log } from "../log.ts";
|
||||
|
||||
/** Where the rule goes. Its own file — the one beside it belongs to something else. */
|
||||
export const RULE = "/etc/dnsmasq.d/mesh-lab.conf";
|
||||
|
||||
export interface Reached {
|
||||
instanceId: string;
|
||||
bridge: string;
|
||||
/** The address the workstation took on that link. */
|
||||
address: string;
|
||||
/** Machine name to the address its names now answer with. */
|
||||
machines: Record<string, string>;
|
||||
}
|
||||
|
||||
export class ConnectError extends Error {}
|
||||
|
||||
/** Run something as root, and say plainly when that is what failed. */
|
||||
function asRoot(argv: string[]): { ok: boolean; said: string } {
|
||||
const ran = spawnSync("sudo", ["-n", ...argv], { encoding: "utf8" });
|
||||
const said = `${ran.stdout ?? ""}${ran.stderr ?? ""}`.trim();
|
||||
if (ran.status !== 0 && /password|not allowed|no tty/i.test(said)) {
|
||||
throw new ConnectError(
|
||||
`this needs root and sudo asked for a password, which there is nowhere to type here.\n` +
|
||||
` Run it yourself: sudo ${argv.join(" ")}`);
|
||||
}
|
||||
return { ok: ran.status === 0, said };
|
||||
}
|
||||
|
||||
/** The one instance standing, or a refusal naming what it found instead. */
|
||||
export async function theOnlyInstance(): Promise<string> {
|
||||
const ids = [...new Set((await taggedInstances()).map((i) => i.instanceId))].sort();
|
||||
if (ids.length === 1) return ids[0]!;
|
||||
if (ids.length === 0) {
|
||||
throw new ConnectError("no scenario is standing, so there is nothing to reach.");
|
||||
}
|
||||
throw new ConnectError(
|
||||
`${ids.length} scenarios are standing and they may hold the same addresses, so there is no ` +
|
||||
`answer to which one you meant: ${ids.join(", ")}.\n` +
|
||||
` Take the others down, or name one — but only one can be reachable at a time.`);
|
||||
}
|
||||
|
||||
/** Every machine in an instance, with the address it has on the given link. */
|
||||
async function addressesOn(instanceId: string, segment: string): Promise<Record<string, string>> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const machine of (await taggedInstances()).filter((i) => i.instanceId === instanceId)) {
|
||||
const said = await incusOk(
|
||||
["exec", machine.name, "--", "sh", "-c",
|
||||
`ip -4 -o addr show | awk '{print $4}' | cut -d/ -f1`], 30_000);
|
||||
for (const address of (said ?? "").split("\n").map((l) => l.trim()).filter(Boolean)) {
|
||||
if (address.startsWith("127.")) continue;
|
||||
// The first non-loopback address on the segment. A machine on two links has the one that
|
||||
// matches this segment's range, which is what the caller asked about.
|
||||
if (!out[machine.machine]) out[machine.machine] = address;
|
||||
}
|
||||
}
|
||||
void segment;
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Names this workstation already answers for, so a scenario cannot quietly shadow one. */
|
||||
function alreadyServed(): string[] {
|
||||
const beside = "/etc/dnsmasq.d/hal-dns.conf";
|
||||
if (!existsSync(beside)) return [];
|
||||
return [...readFileSync(beside, "utf8").matchAll(/^address=\/([^/]+)\//gm)].map((m) => m[1]!);
|
||||
}
|
||||
|
||||
export async function connect(instanceId?: string): Promise<Reached> {
|
||||
const id = instanceId ?? (await theOnlyInstance());
|
||||
|
||||
const link = (await taggedNetworks()).find(
|
||||
(n) => n.instanceId === id && n.kind === "public" && n.cidr.some((c) => !c.includes(":")));
|
||||
if (!link) {
|
||||
throw new ConnectError(
|
||||
`${id} has no public IPv4 segment, so there is nothing for this workstation to join.`);
|
||||
}
|
||||
const range = link.cidr.find((c) => !c.includes(":"))!;
|
||||
const prefix = range.slice(range.lastIndexOf("/") + 1);
|
||||
|
||||
const machines = await addressesOn(id, link.segment);
|
||||
if (Object.keys(machines).length === 0) {
|
||||
throw new ConnectError(`no machine in ${id} has an address yet — is it still coming up?`);
|
||||
}
|
||||
|
||||
// **Refused rather than shadowed.** This workstation already answers for the mesh it really
|
||||
// runs; a scenario machine sharing one of those names would silently take it over, and the
|
||||
// damage would land on the real thing rather than the lab.
|
||||
const clash = Object.keys(machines)
|
||||
.map((m) => `${m}.internal`)
|
||||
.filter((n) => alreadyServed().includes(n));
|
||||
if (clash.length > 0) {
|
||||
throw new ConnectError(
|
||||
`${clash.join(", ")} is already answered on this workstation for something real. ` +
|
||||
`Connecting would point it at the lab instead, which is the wrong thing to break.`);
|
||||
}
|
||||
|
||||
// An address on the link, high in the range so it does not meet what a scenario declares.
|
||||
const base = Object.values(machines)[0]!.split(".").slice(0, 3).join(".");
|
||||
const mine = `${base}.254`;
|
||||
if (Object.values(machines).includes(mine)) {
|
||||
throw new ConnectError(`${mine} is taken by a machine, and that is the address this uses.`);
|
||||
}
|
||||
|
||||
const added = asRoot(["ip", "addr", "add", `${mine}/${prefix}`, "dev", link.name]);
|
||||
if (!added.ok && !/File exists/i.test(added.said)) {
|
||||
throw new ConnectError(`could not take an address on ${link.name}: ${added.said}`);
|
||||
}
|
||||
|
||||
// Everything under a machine's name, answered with that machine. The same shape the mesh's own
|
||||
// resolver writes, because it is answering the same question.
|
||||
const rule = [
|
||||
"# Written by mesh-lab connect. Removed by mesh-lab disconnect.",
|
||||
"# One scenario at a time: these names are only true while that scenario is standing.",
|
||||
...Object.entries(machines).sort()
|
||||
.map(([machine, address]) => `address=/${machine}.internal/${address}`),
|
||||
"",
|
||||
].join("\n");
|
||||
writeFileSync("/tmp/mesh-lab-dns.conf", rule);
|
||||
const placed = asRoot(["cp", "/tmp/mesh-lab-dns.conf", RULE]);
|
||||
if (!placed.ok) throw new ConnectError(`could not write ${RULE}: ${placed.said}`);
|
||||
// **Restart, not reload.** A reload is SIGHUP, and dnsmasq answers that by re-reading its hosts
|
||||
// file and clearing its cache — not its configuration. The rule was written, the reload
|
||||
// reported success, and nothing resolved. Measured: the daemon's start time was nine days old
|
||||
// after a "successful" reload.
|
||||
//
|
||||
// It costs a moment of no name resolution on this workstation, which is the honest price and is
|
||||
// paid again by disconnect.
|
||||
const reloaded = asRoot(["systemctl", "restart", "dnsmasq"]);
|
||||
if (!reloaded.ok) throw new ConnectError(`could not restart dnsmasq: ${reloaded.said}`);
|
||||
|
||||
log.info(`connected to ${id} as ${mine} on ${link.name}`);
|
||||
return { instanceId: id, bridge: link.name, address: mine, machines };
|
||||
}
|
||||
|
||||
export async function disconnect(): Promise<string[]> {
|
||||
const undone: string[] = [];
|
||||
|
||||
if (existsSync(RULE)) {
|
||||
const removed = asRoot(["rm", "-f", RULE]);
|
||||
if (!removed.ok) throw new ConnectError(`could not remove ${RULE}: ${removed.said}`);
|
||||
asRoot(["systemctl", "restart", "dnsmasq"]);
|
||||
undone.push(`removed ${RULE} and reloaded dnsmasq`);
|
||||
}
|
||||
|
||||
// Any address this took, on any lab link still present. Done by looking rather than by
|
||||
// remembering: a workstation that was rebooted, or a scenario destroyed under it, must still
|
||||
// be able to tidy up.
|
||||
for (const link of await taggedNetworks()) {
|
||||
const shown = await incusOk(["network", "info", link.name], 15_000);
|
||||
if (shown === null) continue;
|
||||
const ran = spawnSync("ip", ["-4", "-o", "addr", "show", "dev", link.name], { encoding: "utf8" });
|
||||
for (const line of (ran.stdout ?? "").split("\n")) {
|
||||
const found = line.match(/inet (\d+\.\d+\.\d+\.254\/\d+)/);
|
||||
if (!found) continue;
|
||||
const dropped = asRoot(["ip", "addr", "del", found[1]!, "dev", link.name]);
|
||||
if (dropped.ok) undone.push(`gave up ${found[1]} on ${link.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (undone.length === 0) undone.push("nothing was connected");
|
||||
return undone;
|
||||
}
|
||||
|
||||
/** What is connected now, for a person who cannot remember. */
|
||||
export async function connection(): Promise<string[]> {
|
||||
if (!existsSync(RULE)) return [];
|
||||
return readFileSync(RULE, "utf8").split("\n")
|
||||
.filter((l) => l.startsWith("address=/"))
|
||||
.map((l) => {
|
||||
const [, name, address] = l.match(/^address=\/([^/]+)\/(.+)$/) ?? [];
|
||||
return `${name} → ${address}`;
|
||||
});
|
||||
}
|
||||
|
||||
void incus;
|
||||
@@ -0,0 +1,14 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { RULE } from "../src/lifecycle/connect.ts";
|
||||
|
||||
// The rule goes in its own file, beside the one this workstation already has.
|
||||
//
|
||||
// **Not into it.** The file next to this belongs to the mesh that really runs here, and it is
|
||||
// generated — writing into it would be edited-away at best and would break real name resolution
|
||||
// at worst. novox/hq: never edit a file something else owns.
|
||||
test("the rule is its own file, not the one already there", () => {
|
||||
assert.match(RULE, /^\/etc\/dnsmasq\.d\/mesh-lab\.conf$/);
|
||||
assert.doesNotMatch(RULE, /hal/, "it would be writing into something else's file");
|
||||
});
|
||||
Reference in New Issue
Block a user