novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

The large mesh bed runs end to end again: 21/21 #48

Merged
jschoubben merged 5 commits from multiple-fixes into main 2026-09-22 00:19:16 +00:00
Conversation 0 Commits 5 Files Changed 4
+107 -413
4 changed files with 107 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1 +1,2 @@
node_modules/
node_modules
node_modules
-1
View File
@@ -1 +0,0 @@
/home/jochen/projects/novox/mesh-lab/node_modules
test/integration/mesh-grant-end-to-end.test.ts
+13
View File
@@ -227,4 +227,17 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a
// writing the contributions rather than the bed.
const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`);
assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`);
// A grant means exactly the consumer's own keys — `<login>:*`, the keyspace redis's provisioner
// scopes the ACL user to: under it the consumer reads and writes, outside it and on the server as
// a whole it is refused. Carried over from the large mesh bed's retired cache-grant test —
// without this a provisioner that granted everything would keep every bed green.
const asConsumer = (command: string) =>
on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`);
assert.match((await asConsumer(`SET ${as}:proof yes`)).out, /OK/, "the consumer cannot write under its own login");
assert.match((await asConsumer(`GET ${as}:proof`)).out, /yes/, "the consumer cannot read back what it wrote");
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
"the consumer wrote outside its own keys, so the grant means more than it says");
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
"the consumer flushed the whole server, so the grant means more than it says");
});
test/integration/mesh.test.ts
+93 -412
View File
@@ -18,13 +18,13 @@
import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import { existsSync } from "node:fs";
import { loadScenario } from "../../src/declaration/parse.ts";
import { raise } from "../../src/lifecycle/raise.ts";
import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts";
import type { HeldImage } from "../../src/pinning.ts";
import { destroy, exec } from "../../src/lifecycle/operate.ts";
import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts";
import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts";
import { incus } from "../../src/incus/client.ts";
import { machineName } from "../../src/lifecycle/names.ts";
import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts";
@@ -37,7 +37,6 @@ const binary = hostBinaryPath();
const bundle = process.env["MESH_LAB_BUNDLE"] ?? "";
const builder = process.env["MESH_LAB_BUILDER"] ?? "";
/** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */
const moduleExamples = process.env["MESH_LAB_MODULES"] ?? "";
const skip = !capability.usable
? `lab not usable: ${capability.why}`
@@ -45,7 +44,8 @@ const skip = !capability.usable
? "MESH_LAB_HOST_BINARY is not set to a built mesh-host"
: !bundle || !existsSync(bundle)
? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)"
: false;
// The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074).
: catalogueIsPresent() || false;
const SCENARIO = "two-nodes";
let instanceId = "";
@@ -200,6 +200,24 @@ function tokenFrom(said: string): string {
return found;
}
/** The builder started by hand on the anchor, against the foundation broker's plain port on
* loopback — the one that builds until a builder module can (see the retired test's note). */
async function startBuilder(): Promise<void> {
// The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a
// return, so it is pushed whenever the machine has none.
if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) {
await incus([
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
"--mode", "0755",
], 180_000);
}
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
await must("anchor", `pgrep -x mesh-builder >/dev/null || ` +
`(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`);
}
before(async () => {
if (skip) return;
@@ -232,6 +250,8 @@ before(async () => {
const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`);
assert.equal(running.out.trim(), "yes",
"the host did not come back after a restore, so nothing would apply anything");
// The hand-started builder is memory too, and the snapshot is disk.
if (builder) await startBuilder();
console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` +
`and started the host again`);
@@ -258,17 +278,7 @@ before(async () => {
// A build machine, so anything here can ask the mesh to build something. Placed rather than
// assumed: nothing else in this scenario would start one.
if (builder) {
await incus([
"file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`,
"--mode", "0755",
], 180_000);
await must("anchor", `mkdir -p /var/lib/mesh-builder`);
await must("anchor",
`MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` +
`MESH_WORKSPACE=/var/lib/mesh-builder ` +
`nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`);
}
if (builder) await startBuilder();
if (warming) {
// Snapshotted only now, with everything up: a state worth returning to is the one after the
// part nobody wants to repeat.
@@ -347,10 +357,16 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou
await mesh("assign laptop meshboard");
for (const machine of ["anchor", "laptop"]) {
await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`);
// Once. On a warm return the host is already running (see `before`); a second one would
// consume the same queue and apply the same declaration twice, concurrently.
await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`);
}
await mesh("push");
await new Promise((r) => setTimeout(r, 8000));
// The anchor's derived filter, admitting the hub's port — what genesis does on the control-node,
// and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter
// keeps the hub closed and nothing on the laptop reaches anchor over the private network.
await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on });
const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim();
// Named after the machine *and* the module, because a consumer is both (novox/hq
@@ -618,6 +634,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
`"capabilities":["container-runtime"],` +
`"claims":[{"name":"the-artifact-store","scope":"node"}],` +
`"serves":{"artifact-store":{"port":5000}},` +
`"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` +
`"resources":[` +
`{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` +
`{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` +
@@ -631,10 +648,15 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async (
await mesh("module add /registry.json");
await mesh("assign anchor registry");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 12_000));
// The store's image is pulled from upstream at apply, over the uplink; that takes what it takes.
let names = "";
for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) {
await new Promise((r) => setTimeout(r, 3000));
names = await must("anchor", `docker ps --format '{{.Names}}'`);
}
// Running, and answering — a container that is up is not a registry that replies.
assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/);
assert.match(names, /mesh-registry/,
`the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`);
let answers = false;
for (let i = 0; i < 20 && !answers; i++) {
answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok;
@@ -655,8 +677,11 @@ test("a machine serves its internal name with a certificate the mesh issued", {
// The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity).
// Asserted with a real handshake: a certificate that parses and does not chain fails at the
// moment something connects, which is the worst place to find out.
// The port the handshake below is tried on, declared: the anchor filters what its modules
// did not declare (ADR 0088), and a test server on an undeclared port proves only that.
await must("anchor", `printf %s '{"module":"served","version":"1",` +
`"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` +
`"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` +
`"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` +
`> /tmp/served.json`);
await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`);
@@ -967,7 +992,7 @@ test("a route is a grant: a workload is reached by the name it asked for", {
// they are different questions: one says who may reach it, the other says by what name — and
// the earlier test left this machine filtering, so a module that asked for a route and not for
// the port would be unreachable by the proxy it just asked for.
await must("anchor", `printf %s '{"module":"storefront","version":"1",` +
await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` +
`"requires":["route"],"capabilities":["container-runtime"],` +
`"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` +
`"binds":{"route":"/etc/storefront/route.json"},` +
@@ -1131,6 +1156,10 @@ test("a new commit reaches a machine that is already running the old one", {
// novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the
// question "did my change go out?". This is that question, end to end — a commit, a build, a
// catalogue, and a machine that ends up running what the source says.
// The hand-started builder does not outlive a broker restart, and the foundation's broker is
// recreated when the first push reconciles it: a builder is (re)started here, where a build is
// asked for. The mesh's own builder is a module with a restart policy and needs none of this.
await startBuilder();
const repo = "/var/lib/mesh/builder/repositories/delivered";
const write = async (what: string) =>
await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` +
@@ -1258,79 +1287,11 @@ test("the board names the machine that is not doing what it was told", {
});
// Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries.
test("the hub can be filtered without severing the mesh", {
skip, timeout: 900_000,
}, async () => {
// The machine that most needs a firewall was the one that could not have one. A hub is dialled
// by every node at other sites; a machine that is not a hub dials out and needs nothing open.
// They are the same module, so a static `listens` cannot say it — and the machine it gets wrong
// is the one facing the public internet.
//
// The failure this guards against is not subtle and is very hard to recover from: a rule set
// that closes the hub's own port takes the private network down, and the mesh's way of fixing
// anything is to send a declaration over it.
// Its own directory. Another module on this machine already declares /etc/mesh, and the mesh
// refuses two modules declaring one path rather than letting the second quietly win — which it
// did here, correctly, the first time this ran.
const rules = "/etc/mesh-hub/filter.nft";
await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` +
`"capabilities":["firewall"],` +
`"filtering":{"into":"${rules}"},` +
`"resources":[{"id":"nftables","type":"package","package":"nftables"},` +
`{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` +
`{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` +
`"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` +
`[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` +
`ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` +
`{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` +
`"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`);
await must("anchor", `docker cp /tmp/hubfilter.json mesh-controller:/hubfilter.json`);
await mesh("module add /hubfilter.json");
await mesh("assign anchor hubfilter");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 20_000));
// The hub's own way onto the private network is open, and derived — nothing in that manifest
// mentions a port.
const written = await must("anchor", `cat ${rules}`);
assert.match(written, /udp dport 51820 accept/,
`the hub's rule set closes the private network it is the way onto:\n${written}`);
// The module that provides the private network, not the requirement it answers: `networking`
// is the domain a module offers, and what caused a rule is the module itself.
assert.match(written, /# mesh-wireguard — the private network/,
`the rule does not name what caused it:\n${written}`);
// Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if
// the overlay is severed. This is the assertion that matters — a rule file that looks right and
// a mesh that has stopped are exactly what this is guarding against.
assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/);
await must("laptop", `rm -f /etc/mesh-still-works`);
await must("anchor", `printf %s '{"module":"stillworks","version":"1",` +
`"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` +
`"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`);
await must("anchor", `docker cp /tmp/stillworks.json mesh-controller:/stillworks.json`);
await mesh("module add /stillworks.json");
await mesh("assign laptop stillworks");
await mesh("push laptop");
let arrived = false;
for (let i = 0; i < 20 && !arrived; i++) {
arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok;
if (!arrived) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(arrived,
"the hub applied its own rule set and the mesh stopped reaching the other machine");
// And the other machine still reaches the hub over the private network, which is what the
// opened port is for.
assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok,
"the private network is down after the hub filtered itself");
await mesh("unassign anchor hubfilter");
await mesh("unassign laptop stillworks");
await mesh("push");
});
// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on
// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a
// filter module derives the rules — so the credential test above assigns the catalogue's and
// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh
// was not severed. Retired 2026-09-22.
test("a container reaches another machine by the name the mesh gave it", {
skip, timeout: 900_000,
@@ -1375,6 +1336,23 @@ test("a container reaches another machine by the name the mesh gave it", {
// Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being
// told each one.
/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so
* it is issued once per machine. The mesh writes the resolver's data as a fact the module
* declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */
const resolverIssued = new Set<string>();
async function resolverModules(machines: string[]): Promise<void> {
for (const name of ["dnsmasq", "resolved-split-dns"]) {
const manifest = catalogueModule(name, held);
await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
for (const machine of machines) {
if (resolverIssued.has(machine)) continue;
await mesh(`module issue dnsmasq --node ${machine}`);
resolverIssued.add(machine);
}
}
test("every name under a machine resolves to that machine", {
skip, timeout: 900_000,
}, async () => {
@@ -1385,9 +1363,11 @@ test("every name under a machine resolves to that machine", {
//
// The mesh writes the data and runs no daemon: a resolver is third-party software, and the
// mesh has no business choosing one. So what is checked here is the mesh's half — that the
// data is right, complete, and follows the machines.
await mesh("assign anchor mesh-resolver");
await mesh("assign laptop mesh-resolver");
// data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq
// declares, so that module is what is assigned; what it runs is the next test's concern.
await resolverModules(["anchor", "laptop"]);
await mesh("assign anchor dnsmasq");
await mesh("assign laptop dnsmasq");
await mesh("push");
await new Promise((r) => setTimeout(r, 15_000));
@@ -1413,10 +1393,10 @@ test("every name under a machine resolves to that machine", {
// because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name
// fails at once and says which name it was.
//
// Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the
// network, so unassigning the domain module alone leaves the machine on the network — pulled
// back by its own requirement. The mesh was right and this test was wrong the first time.
await mesh("unassign laptop mesh-resolver");
// Both: the resolver's data follows the private network, so the machine leaves the network as
// well as the resolver, and what is asserted is that the machine that stayed is answered for and
// the one that left is not.
await mesh("unassign laptop dnsmasq");
await mesh("unassign laptop networking");
await mesh("push anchor");
await new Promise((r) => setTimeout(r, 15_000));
@@ -1428,14 +1408,13 @@ test("every name under a machine resolves to that machine", {
`the machine that stayed lost its own name:\n${after}`);
await mesh("assign laptop networking");
await mesh("unassign anchor mesh-resolver");
await mesh("unassign anchor dnsmasq");
await mesh("push");
await new Promise((r) => setTimeout(r, 15_000));
});
test("a service is reached by a name under the machine it runs on", {
skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-controller's examples/modules" : false),
timeout: 900_000,
skip, timeout: 900_000,
}, async () => {
// postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is
// the node, so anything under a node's name must resolve to that node. What routes it once it
@@ -1447,12 +1426,7 @@ test("a service is reached by a name under the machine it runs on", {
// /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a
// refusal rather than a fight over the file — and picking the wrong one here would have been
// testing that fight.
for (const name of ["dnsmasq", "resolved-split-dns"]) {
const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8");
await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`);
await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
await resolverModules(["anchor", "laptop"]);
// Both machines, because a node resolves from its own copy — the same rule as everything else
// it holds. A mesh where one machine answers for all of them stops resolving when that machine
@@ -1679,305 +1653,12 @@ test("a third-party workload is adopted, with the credential it already had", {
// Running them needs their images stocked and two provisioners built, which is a separate and
// larger job. This is the half that can be known now, and it is the half where a design fault
// would live.
test("the real modules resolve together, and compose a declaration a host accepts", {
skip, timeout: 300_000,
}, async (t) => {
// Everything the catalogue holds, except two whose names this mesh is already running under:
// `registry` is the artifact store the suite stood up, and `umami` is the adopted workload —
// adding the catalogue's manifests would replace the records of modules that are live and
// assigned, and the adopted umami would suddenly require a database it never asked for.
const modules = ["postgres", "keycloak", "gitea", "minio", "mailu",
"redis", "grafana", "nextcloud", "searxng", "influxdb", "verdaccio"];
const planned: string[] = [];
for (const name of modules) {
const raw = readFileSync(
`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
// Pointed at this scenario's registry before being added, exactly as the forge is.
//
// **Not cosmetic.** Some of these name an image the mesh builds, whose digest does not exist
// until it is built — so the file legitimately carries a placeholder, and composing a
// declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is
// what this test used to do.
const pinned = pinnedInto(raw, held);
// What this scenario does not serve cannot be redirected, and a module still naming a
// placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped
// and said, rather than silently dropped: a planning test quietly covering four modules
// instead of five is the false coverage this suite exists to prevent.
const left = stillUnpinned(pinned);
if (left.length > 0) {
console.log(`skipping ${name}: this scenario serves no ${left.join(", ")}`);
continue;
}
planned.push(name);
await must("anchor", `printf %s ${quote(pinned)} > /${name}.json`);
await must("anchor", `docker cp /${name}.json mesh-controller:/${name}.json`);
await mesh(`module add /${name}.json`);
}
// **Put the machine back whatever happens.** Tests here share one mesh, so what this one
// assigns is what the next one inherits. Written at the end of the body once, it was skipped
// the first time this test failed — and the next test's push was refused by a module this one
// had left behind, which reads as a fault in the test that was actually working.
t.after(async () => {
for (const name of planned) await mesh(`unassign anchor ${name}`).catch(() => {});
});
// Assigned one at a time, because assignment resolves the whole set and says so immediately.
// A refusal here is the graph rejecting something, which is the point of asking.
for (const name of planned) {
await mesh(`assign anchor ${name}`);
}
const plan = await mesh("plan anchor --json", 120_000);
const declaration = JSON.parse(plan.slice(plan.indexOf("{")));
const byId = new Map<string, any>(
(declaration.resources as any[]).map((r) => [r.id, r]));
const ids = [...byId.keys()];
// Every module's own network, which only exists because more than one container needs to reach
// another by name.
for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) {
assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`);
assert.equal(byId.get(id).type, "network");
}
// The cross-module edge: keycloak asked for a database and was told where it is and given a
// credential. Neither file is anything keycloak's manifest could have written.
const bound = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.json");
assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`);
assert.match(JSON.stringify(bound), /postgres/,
"keycloak's binding does not name what answered its requirement");
// The password, alone in a file and sealed. It is a password and nothing else, so nothing reads
// it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction.
const credential = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.secret");
assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`);
assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it");
assert.ok(!credential.content, "a credential arrived as content rather than sealed");
// And the connection itself, which keycloak could not have written: the address and port come
// from what the provider serves, and the user name from what the mesh decided both ends would
// call this consumer (novox/hq 04-ISSUES/023).
const connection = [...byId.values()].find((r) =>
r.type === "file" && r.path === "/var/lib/keycloak/database.env");
assert.ok(connection, "keycloak was given no database configuration");
assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/,
`keycloak was not told what name to present:\n${connection.content}`);
assert.doesNotMatch(connection.content, /\$\{bound:/,
`a placeholder reached the machine as a value:\n${connection.content}`);
// The password is the one hole left open, and the sealed value travels beside it. The mesh
// discarded the plaintext, so the host is the only thing that can close it.
assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/,
`the password was not left for the host to fill:\n${connection.content}`);
assert.ok(connection.secrets?.["postgres-database"],
"the sealed credential did not travel with the file that needs it");
assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/,
"the credential was written into the configuration in the clear");
// And the provider was told who asked, which is what its provisioner reconciles against.
const grants = [...byId.values()].find((r) =>
r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants"));
assert.ok(grants, "postgres was never told which modules were granted a database");
assert.match(JSON.stringify(grants), /keycloak|gitea/,
"the grants file names neither module that asked for a database");
// Secrets reach containers as files, never as environment in the declaration.
const containers = [...byId.values()].filter((r) => r.type === "container");
assert.ok(containers.length >= 12,
`only ${containers.length} containers; mailu alone is nine`);
for (const c of containers) {
for (const [key, value] of Object.entries(c.env ?? {})) {
// **An absolute path is a reference to a secret, not a secret**, and naming one is the
// whole design: the mesh delivers a credential as a file and a module says where.
//
// Excluded because `/` is in the base64 alphabet, so any path of 24 characters or more
// matched — `MESH_BROKER_FILE=/var/lib/mesh/builder/broker` was reported as a credential
// the broker would see. A check that fires on the right shape for the wrong reason is
// worse than none: it is the one that gets suppressed, and then it is not there when it
// is right.
if (String(value).startsWith("/")) continue;
assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/,
`${c.name} carries something secret-shaped in env.${key}, which the broker would see`);
}
}
});
// The first of the real module descriptions to actually run.
//
// **Everything before this stopped at composing a declaration.** That proves the control plane and
// the host agree, and proves nothing about whether the thing described works — which is how five
// modules sat pinned to images that did not exist, parsing and resolving perfectly
// (novox/hq 04-ISSUES/025).
//
// The forge is the one worth running first. It needs a database from another module, a password it
// did not choose, and a connection string it could not have written itself: the address and port
// come from what the database serves, and the user name from what the mesh decided both ends would
// call it (04-ISSUES/022 and 023). If any of that is wrong it cannot start, and nothing else in
// this file would notice.
test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 }, async () => {
for (const name of ["postgres", "gitea"]) {
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8");
// An image the mesh builds has no digest until it is built, and one it does not build belongs
// to whichever registry served it. Only the first is rewritten; the second is pulled.
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
`${name} still names an image nothing serves, so it could not start`);
await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`);
await must("anchor", `docker cp /run-${name}.json mesh-controller:/run-${name}.json`);
await mesh(`module add /run-${name}.json`);
await mesh(`assign anchor ${name}`);
}
await mesh("push anchor", 300_000);
// **What the machine says it did, before asking what it produced.** This test pushed and then
// waited for a database role, so when the containers were never created at all it reported "no
// login was created" — true, and silent about the reason. A push that was accepted and an apply
// that worked are different facts, and the second is the one this depends on.
//
// And waited for, because `push` sends without waiting. Reading `status` the instant it returns
// describes the apply *before* this one, which is how this test came to report a missing
// container while insisting the machine was fine.
await settled("anchor");
const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out;
// Named with what the mesh meant to send, not only with what the machine has. A container that
// is absent because the mesh never asked for it and one that is absent because the machine could
// not make it are the same sentence here and different faults entirely, and the plan is the only
// thing that tells them apart.
assert.match(running, /\bpostgres\b/,
`the database module was pushed and no container for it exists:\n${running}\n\n` +
`what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` +
`${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`);
// The database first: until the provisioner has made the login, the forge has nothing to
// connect to and its own start would prove only that it retries.
const psql = async (q: string) =>
(await on("anchor",
`docker exec postgres psql -U postgres -qAt -c ${quote(q)}`, 60_000)).out.trim();
// Both halves in one poll. The provisioner makes the role and then the database, and a test
// that waited for the first and checked the second once was racing the gap between two
// statements — it lost, once, eighteen seconds into a run.
let made = "";
for (let i = 0; i < 40 && made !== "t"; i++) {
made = await psql("select true from pg_roles where rolname = 'mesh_anchor_gitea'" +
" and exists (select from pg_database where datname = 'gitea')");
if (made !== "t") await new Promise((r) => setTimeout(r, 3000));
}
assert.equal(made, "t",
`no login was created for the forge:\n${(await on("anchor", "docker logs mesh-provision-postgres 2>&1 | tail -20")).out}`);
// And the forge itself, answering. Not that its container exists — that it serves.
//
// On the port the mesh assigned, not the one the module declared (novox/hq ADR 0038): the
// module says 3000 and the machine publishes wherever the mesh put it. Read from the plan,
// because the plan is the same composition a push sends.
const planned = await mesh("plan anchor --json", 120_000);
const mapping = (JSON.parse(planned.slice(planned.indexOf("{"))).resources as any[])
.find((r) => r.id === "gitea.server")?.ports
?.map(String).find((p: string) => p.endsWith(":3000"));
assert.ok(mapping, "the plan does not say where the machine publishes the forge");
const at = mapping.split(":")[0];
let answered = false;
let said = { out: "", ok: false };
for (let i = 0; i < 60 && !answered; i++) {
said = await on("anchor", `curl -sf -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${at}/`, 30_000);
answered = said.out.trim().startsWith("2") || said.out.trim() === "303";
if (!answered) await new Promise((r) => setTimeout(r, 5000));
}
assert.ok(answered,
`the forge never answered (last: ${said.out.trim()}):\n` +
`${(await on("anchor", "docker logs gitea 2>&1 | tail -25")).out}`);
// **The credential actually worked.** A forge that started and could not reach its database
// would still answer on its port, so the log is where the difference lives.
const log = (await on("anchor", "docker logs gitea 2>&1 | tail -60")).out;
assert.doesNotMatch(log, /password authentication failed|connection refused|does not exist/i,
`the forge started and could not use the database it was given:\n${log}`);
await mesh("unassign anchor gitea");
await mesh("unassign anchor postgres");
await mesh("push anchor", 300_000);
});
test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600_000 }, async (t) => {
// The third provision after a database and a bucket, and the first whose tenancy is enforced
// by the store's own ACL rather than by separate namespaces: every consumer shares one
// keyspace, so the grant is a pattern — and the test is that the pattern means what the
// manifest said, in both directions.
const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8");
const pinned = pinnedInto(raw, held);
assert.deepEqual(stillUnpinned(pinned), [],
"redis still names an image nothing serves, so it could not start");
await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`);
await must("anchor", `docker cp /run-redis.json mesh-controller:/run-redis.json`);
await mesh("module add /run-redis.json");
// A consumer with no container: what is under test is the credential's reach, and files on the
// machine are enough to prove it — the same reduction the first credential test makes.
await must("anchor", `printf %s '{"module":"cachetest","version":"1",` +
`"requires":["redis-cache"],` +
`"contributes":{"redis-cache":{"prefix":"cachetest"}},` +
`"binds":{"redis-cache":"/var/lib/cachetest/cache.json"},` +
`"secrets":{"redis-cache":"/var/lib/cachetest/cache.secret"},` +
`"resources":[{"id":"state","type":"directory","path":"/var/lib/cachetest","mode":"0700"}]}' ` +
`> /cachetest.json`);
await must("anchor", `docker cp /cachetest.json mesh-controller:/cachetest.json`);
await mesh("module add /cachetest.json");
await mesh("assign anchor redis");
await mesh("assign anchor cachetest");
await mesh("push anchor", 300_000);
await settled("anchor");
t.after(async () => {
for (const name of ["cachetest", "redis"]) {
await mesh(`unassign anchor ${name}`).catch(() => {});
}
await mesh("push anchor", 300_000).catch(() => {});
});
// What the mesh told each end. The consumer's user name comes from its binding; the user's
// password from the sealed file beside it — both written by the host, neither invented here.
const bound = JSON.parse(await must("anchor", `cat /var/lib/cachetest/cache.json`));
const user = bound.as;
assert.ok(user?.startsWith("mesh_"), `the binding does not carry a usable user: ${user}`);
const secret = (await must("anchor", `cat /var/lib/cachetest/cache.secret`)).trim();
// The provisioner has to have run before anything can authenticate. Waited for via the store
// itself: the user list, asked with the server's own password, which the conf file the host
// wrote holds on the machine.
const admin = (await must("anchor",
`awk '/^requirepass/ {print $2}' /var/lib/redis-module/redis.conf`)).trim();
let granted = false;
for (let i = 0; i < 40 && !granted; i++) {
const users = (await on("anchor",
`docker exec redis redis-cli --no-auth-warning -a ${quote(admin)} ACL USERS`)).out;
granted = users.includes(user);
if (!granted) await new Promise((r) => setTimeout(r, 3000));
}
assert.ok(granted, `no user was created for the consumer:
` +
`containers:\n${(await on("anchor", "docker ps -a --format '{{.Names}} {{.Status}}' | head -20")).out}\n` +
`the store:\n${(await on("anchor", "docker logs redis 2>&1 | tail -15")).out}\n` +
`the provisioner:\n${(await on("anchor", "docker logs mesh-provision-redis 2>&1 | tail -15")).out}`);
const asConsumer = (command: string) =>
on("anchor", `docker exec redis redis-cli --no-auth-warning ` +
`--user ${quote(user)} --pass ${quote(secret)} ${command}`);
// Its own keys: usable.
assert.match((await asConsumer("SET cachetest:proof yes")).out, /OK/,
"the consumer cannot write under the prefix it was granted");
assert.match((await asConsumer("GET cachetest:proof")).out, /yes/,
"the consumer cannot read back what it wrote");
// Anyone else's: refused by the store itself, which is the entire point of the grant.
assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i,
"the consumer wrote outside its prefix — the grant means more than the manifest said");
assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,
"the consumer can flush the store, which no tenant may");
});
// Three tests lived here that read the mesh's example modules, which moved to the catalogue.
// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood
// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the
// catalogue's modules together and its gate is the composed declaration accepted and every core
// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates
// on gitea running but does not yet ask it to answer on its port with the credential it was given,
// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy
// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into
// mesh-grant-end-to-end, against the catalogue's redis.