A bed that raises the foundation from the bundle derives the anchor's filter before it relies on the hub
The base ruleset (ADR 0088) admits ssh, the bus and the registry and nothing else until the mesh derives one, and the mesh derives one only where the filter module is assigned — which genesis does and these beds did not. Without it the hub's WireGuard port stayed closed, no joined node's tunnel formed, and every module dialling the anchor by its overlay name timed out fetching the broker's certificate; the model-usage bed showed it as a login that failed for a role never made.
This commit is contained in:
@@ -38,7 +38,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -286,6 +286,9 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot
|
|||||||
await mesh(`overlay place ${NODE} --site lab`);
|
await mesh(`overlay place ${NODE} --site lab`);
|
||||||
await mesh("assign anchor networking");
|
await mesh("assign anchor networking");
|
||||||
await mesh(`assign ${NODE} networking`);
|
await mesh(`assign ${NODE} networking`);
|
||||||
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||||
|
// and what a bed raised from the bundle must do itself (ADR 0088; see the harness).
|
||||||
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
||||||
|
|
||||||
await addIssueAssign("postgres", postgresManifest);
|
await addIssueAssign("postgres", postgresManifest);
|
||||||
await addIssueAssign("model-usage", modelUsageManifest);
|
await addIssueAssign("model-usage", modelUsageManifest);
|
||||||
|
|||||||
@@ -9,7 +9,8 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { readFileSync } from "node:fs";
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
|
import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts";
|
||||||
import { destroy, list } from "../../src/lifecycle/operate.ts";
|
import { destroy, list } from "../../src/lifecycle/operate.ts";
|
||||||
import { diagramFromLive } from "../../src/diagram/from-live.ts";
|
import { diagramFromLive } from "../../src/diagram/from-live.ts";
|
||||||
@@ -230,3 +231,57 @@ export async function assertUniversalInvariants(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- the packet filter, where a bed raises the foundation without genesis ------------------------
|
||||||
|
|
||||||
|
/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */
|
||||||
|
export const FILTER_MODULE = "nftables";
|
||||||
|
|
||||||
|
/** The catalogue's manifest for a module, under either spelling of MESH_LAB_CATALOG: the modules
|
||||||
|
* directory, or the checkout that holds it. */
|
||||||
|
export function catalogueManifest(module: string): string {
|
||||||
|
const dir = process.env["MESH_LAB_CATALOG"] ?? "";
|
||||||
|
for (const candidate of [resolve(dir, module, "module.json"), resolve(dir, "modules", module, "module.json")]) {
|
||||||
|
if (existsSync(candidate)) return candidate;
|
||||||
|
}
|
||||||
|
throw new Error(`no manifest for ${module} under MESH_LAB_CATALOG=${dir || "(unset)"}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shellQuote(s: string): string {
|
||||||
|
return `'${s.replaceAll("'", `'\\''`)}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The foundation is raised behind a base ruleset that admits ssh, the bus and the registry and
|
||||||
|
* nothing else, "until the mesh derives one" (novox/hq ADR 0088) — and the mesh derives one only
|
||||||
|
* where the packet-filter module is assigned, which genesis does on the control-node. A bed that
|
||||||
|
* raises the foundation from the bundle skips genesis, so it must do the same before it relies on
|
||||||
|
* an overlay hub there: the hub's port is derived from its endpoint, and until the derived ruleset
|
||||||
|
* lands no joined node's tunnel forms, and every module that dials the anchor by its overlay name
|
||||||
|
* times out fetching the broker's certificate — the failure this helper was written after.
|
||||||
|
*
|
||||||
|
* Registered, assigned, pushed, and then WAITED FOR: what the machine loaded must admit the hub's
|
||||||
|
* port, which the base ruleset cannot. Call it after the hub is placed, so there is a port to derive.
|
||||||
|
*/
|
||||||
|
export async function deriveTheFilterOn(o: {
|
||||||
|
machine: string; node: string; hubPort: number;
|
||||||
|
must: (machine: string, command: string, timeoutMs?: number) => Promise<string>;
|
||||||
|
mesh: (command: string, timeoutMs?: number) => Promise<string>;
|
||||||
|
on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>;
|
||||||
|
}): Promise<string> {
|
||||||
|
const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8");
|
||||||
|
await o.must(o.machine,
|
||||||
|
`printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`);
|
||||||
|
await o.mesh(`module add /${FILTER_MODULE}.json`);
|
||||||
|
await o.mesh(`assign ${o.node} ${FILTER_MODULE}`);
|
||||||
|
await o.mesh(`push ${o.node}`, 600_000);
|
||||||
|
const admits = new RegExp(`udp dport ${o.hubPort} accept`);
|
||||||
|
const deadline = Date.now() + 180_000;
|
||||||
|
let ruleset = "";
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
ruleset = (await o.on(o.machine, `nft list table inet mesh 2>&1`)).out;
|
||||||
|
if (admits.test(ruleset)) return ruleset;
|
||||||
|
await new Promise((r) => setTimeout(r, 5_000));
|
||||||
|
}
|
||||||
|
assert.fail(`${FILTER_MODULE} is assigned to ${o.node} and the ruleset it loaded does not admit the hub's udp/${o.hubPort}:\n${ruleset}`);
|
||||||
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -301,6 +301,9 @@ test("the lavinmq provider and its consumer ride laptop while the foundation bro
|
|||||||
await mesh(`overlay place ${NODE} --site lab`);
|
await mesh(`overlay place ${NODE} --site lab`);
|
||||||
await mesh("assign anchor networking");
|
await mesh("assign anchor networking");
|
||||||
await mesh(`assign ${NODE} networking`);
|
await mesh(`assign ${NODE} networking`);
|
||||||
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||||
|
// and what a bed raised from the bundle must do itself (ADR 0088; see the harness).
|
||||||
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
||||||
|
|
||||||
await addIssueAssign("lavinmq", lavinmqManifest);
|
await addIssueAssign("lavinmq", lavinmqManifest);
|
||||||
await addIssueAssign("amqp-ping", amqpPingManifest);
|
await addIssueAssign("amqp-ping", amqpPingManifest);
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ import { loadScenario } from "../../src/declaration/parse.ts";
|
|||||||
import { raise } from "../../src/lifecycle/raise.ts";
|
import { raise } from "../../src/lifecycle/raise.ts";
|
||||||
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
import { destroy, exec } from "../../src/lifecycle/operate.ts";
|
||||||
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
|
||||||
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
|
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts";
|
||||||
import type { HeldImage } from "../../src/pinning.ts";
|
import type { HeldImage } from "../../src/pinning.ts";
|
||||||
|
|
||||||
const capability = await labIsUsable();
|
const capability = await labIsUsable();
|
||||||
@@ -284,6 +284,9 @@ test("the whole ace service set resolves, installs and converges on one node in
|
|||||||
await mesh(`overlay place ${NODE} --site lab`);
|
await mesh(`overlay place ${NODE} --site lab`);
|
||||||
await mesh("assign anchor networking");
|
await mesh("assign anchor networking");
|
||||||
await mesh(`assign ${NODE} networking`);
|
await mesh(`assign ${NODE} networking`);
|
||||||
|
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
|
||||||
|
// and what a bed raised from the bundle must do itself (ADR 0088; see the harness).
|
||||||
|
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
|
||||||
|
|
||||||
// Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one
|
// Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one
|
||||||
// bad assignment cannot poison the whole-node push.
|
// bad assignment cannot poison the whole-node push.
|
||||||
|
|||||||
Reference in New Issue
Block a user