From 9eaa3a623d054a5e9cf1d2f09d08a2d497752d15 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 00:38:07 +0200 Subject: [PATCH 1/5] node_modules is not tracked: the link committed by mistake goes, and the ignore covers a link too --- .gitignore | 1 + node_modules | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 node_modules diff --git a/.gitignore b/.gitignore index c2658d7..e768632 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules/ +node_modules diff --git a/node_modules b/node_modules deleted file mode 120000 index 34fd1f4..0000000 --- a/node_modules +++ /dev/null @@ -1 +0,0 @@ -/home/jochen/projects/novox/mesh-lab/node_modules \ No newline at end of file From 353cf88a4be31d2de1abde801c426d16d72f1976 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 01:16:13 +0200 Subject: [PATCH 2/5] The large mesh bed catches up with the mesh: the anchor's filter derived, ports declared, an image awaited, one host and builder on a warm return, resolvers from the catalogue; four tests retired for the beds that prove them --- test/integration/mesh.test.ts | 475 ++++++---------------------------- 1 file changed, 73 insertions(+), 402 deletions(-) diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index c8e5eeb..245fe70 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -18,13 +18,13 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; -import { pinnedInto, stillUnpinned, type HeldImage } from "../../src/pinning.ts"; +import type { HeldImage } from "../../src/pinning.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn } from "./harness.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts"; @@ -37,7 +37,6 @@ const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; const builder = process.env["MESH_LAB_BUILDER"] ?? ""; /** mesh-controller's `examples/modules`, so the manifests proven here are the ones that ship. */ -const moduleExamples = process.env["MESH_LAB_MODULES"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` @@ -200,6 +199,16 @@ function tokenFrom(said: string): string { return found; } +/** The builder started by hand on the anchor, against the foundation broker's plain port on + * loopback — the one that builds until a builder module can (see the retired test's note). */ +async function startBuilder(): Promise { + await must("anchor", `mkdir -p /var/lib/mesh-builder`); + await must("anchor", `pgrep -x mesh-builder >/dev/null || ` + + `(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` + + `MESH_WORKSPACE=/var/lib/mesh-builder ` + + `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3)`); +} + before(async () => { if (skip) return; @@ -232,6 +241,8 @@ before(async () => { const running = await on("anchor", `pgrep -x mesh-host >/dev/null && echo yes || echo no`); assert.equal(running.out.trim(), "yes", "the host did not come back after a restore, so nothing would apply anything"); + // The hand-started builder is memory too, and the snapshot is disk. + if (builder) await startBuilder(); console.log(`warm: returned ${instanceId} to its state in ${seconds.toFixed(1)}s, ` + `and started the host again`); @@ -263,11 +274,7 @@ before(async () => { "file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`, "--mode", "0755", ], 180_000); - await must("anchor", `mkdir -p /var/lib/mesh-builder`); - await must("anchor", - `MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` + - `MESH_WORKSPACE=/var/lib/mesh-builder ` + - `nohup /usr/local/bin/mesh-builder > /var/log/mesh-builder.log 2>&1 & sleep 3`); + await startBuilder(); } if (warming) { // Snapshotted only now, with everything up: a state worth returning to is the one after the @@ -347,10 +354,16 @@ test("a credential reaches both ends and the mesh holds neither", { skip, timeou await mesh("assign laptop meshboard"); for (const machine of ["anchor", "laptop"]) { - await must(machine, `nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3`); + // Once. On a warm return the host is already running (see `before`); a second one would + // consume the same queue and apply the same declaration twice, concurrently. + await must(machine, `pgrep -x mesh-host >/dev/null || (nohup ${HOST_PATH} run > /var/log/mesh-host.log 2>&1 & sleep 3)`); } await mesh("push"); await new Promise((r) => setTimeout(r, 8000)); + // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, + // and what a bed raised from the bundle must do itself (ADR 0088). Until it is, the base filter + // keeps the hub closed and nothing on the laptop reaches anchor over the private network. + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); const onConsumer = (await must("laptop", `cat /etc/meshboard/database.password`)).trim(); // Named after the machine *and* the module, because a consumer is both (novox/hq @@ -618,6 +631,7 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async ( `"capabilities":["container-runtime"],` + `"claims":[{"name":"the-artifact-store","scope":"node"}],` + `"serves":{"artifact-store":{"port":5000}},` + + `"listens":[{"port":5000,"from":"mesh","why":"every machine pulls what the mesh built"}],` + `"resources":[` + `{"id":"state","type":"directory","path":"/var/lib/mesh/registry","mode":"0700"},` + `{"id":"store","type":"container","name":"mesh-registry","image":"${ARTIFACT_STORE}",` + @@ -631,10 +645,15 @@ test("the mesh runs its own artifact store", { skip, timeout: 900_000 }, async ( await mesh("module add /registry.json"); await mesh("assign anchor registry"); await mesh("push anchor"); - await new Promise((r) => setTimeout(r, 12_000)); - + // The store's image is pulled from upstream at apply, over the uplink; that takes what it takes. + let names = ""; + for (let i = 0; i < 60 && !/mesh-registry/.test(names); i++) { + await new Promise((r) => setTimeout(r, 3000)); + names = await must("anchor", `docker ps --format '{{.Names}}'`); + } // Running, and answering — a container that is up is not a registry that replies. - assert.match(await must("anchor", `docker ps --format '{{.Names}}'`), /mesh-registry/); + assert.match(names, /mesh-registry/, + `the mesh's registry never started:\n${names}\n--- host log ---\n${(await on("anchor", `tail -20 /var/log/mesh-host.log`)).out}`); let answers = false; for (let i = 0; i < 20 && !answers; i++) { answers = (await on("anchor", `curl -sf http://127.0.0.1:5000/v2/ -o /dev/null`)).ok; @@ -655,8 +674,11 @@ test("a machine serves its internal name with a certificate the mesh issued", { // The mesh's own authority certifies names only the mesh knows (novox/hq 08-connectivity). // Asserted with a real handshake: a certificate that parses and does not chain fails at the // moment something connects, which is the worst place to find out. + // The port the handshake below is tried on, declared: the anchor filters what its modules + // did not declare (ADR 0088), and a test server on an undeclared port proves only that. await must("anchor", `printf %s '{"module":"served","version":"1",` + `"certificate":{"into":"/etc/mesh/serving.crt","authority":"/etc/mesh/authority.crt"},` + + `"listens":[{"port":8443,"from":"mesh","why":"a handshake against the certificate the mesh issued"}],` + `"resources":[{"id":"dir","type":"directory","path":"/etc/mesh","mode":"0755"}]}' ` + `> /tmp/served.json`); await must("anchor", `docker cp /tmp/served.json mesh-controller:/served.json`); @@ -967,7 +989,7 @@ test("a route is a grant: a workload is reached by the name it asked for", { // they are different questions: one says who may reach it, the other says by what name — and // the earlier test left this machine filtering, so a module that asked for a route and not for // the port would be unreachable by the proxy it just asked for. - await must("anchor", `printf %s '{"module":"storefront","version":"1",` + + await must("anchor", `printf %s '{"module":"storefront","version":"1","slug":"shop",` + `"requires":["route"],"capabilities":["container-runtime"],` + `"contributes":{"route":{"name":"shop.mesh.test","port":8088}},` + `"binds":{"route":"/etc/storefront/route.json"},` + @@ -1258,79 +1280,11 @@ test("the board names the machine that is not doing what it was told", { }); // Defends novox/hq ADR 0007: filtering the hub must not cut the overlay it carries. -test("the hub can be filtered without severing the mesh", { - skip, timeout: 900_000, -}, async () => { - // The machine that most needs a firewall was the one that could not have one. A hub is dialled - // by every node at other sites; a machine that is not a hub dials out and needs nothing open. - // They are the same module, so a static `listens` cannot say it — and the machine it gets wrong - // is the one facing the public internet. - // - // The failure this guards against is not subtle and is very hard to recover from: a rule set - // that closes the hub's own port takes the private network down, and the mesh's way of fixing - // anything is to send a declaration over it. - // Its own directory. Another module on this machine already declares /etc/mesh, and the mesh - // refuses two modules declaring one path rather than letting the second quietly win — which it - // did here, correctly, the first time this ran. - const rules = "/etc/mesh-hub/filter.nft"; - await must("anchor", `printf %s '{"module":"hubfilter","version":"1",` + - `"capabilities":["firewall"],` + - `"filtering":{"into":"${rules}"},` + - `"resources":[{"id":"nftables","type":"package","package":"nftables"},` + - `{"id":"dir","type":"directory","path":"/etc/mesh-hub","mode":"0755"},` + - `{"id":"unit","type":"file","path":"/etc/systemd/system/hub-filter.service",` + - `"mode":"0644","content":"[Unit]\\nDescription=What the mesh computed for the hub\\n` + - `[Service]\\nType=oneshot\\nRemainAfterExit=yes\\n` + - `ExecStart=/usr/bin/nft -f ${rules}\\n[Install]\\nWantedBy=multi-user.target\\n"},` + - `{"id":"filter","type":"service","unit":"hub-filter.service","state":"running",` + - `"boot":"enabled","restart-on":["filtering"]}]}' > /tmp/hubfilter.json`); - await must("anchor", `docker cp /tmp/hubfilter.json mesh-controller:/hubfilter.json`); - await mesh("module add /hubfilter.json"); - await mesh("assign anchor hubfilter"); - await mesh("push anchor"); - await new Promise((r) => setTimeout(r, 20_000)); - - // The hub's own way onto the private network is open, and derived — nothing in that manifest - // mentions a port. - const written = await must("anchor", `cat ${rules}`); - assert.match(written, /udp dport 51820 accept/, - `the hub's rule set closes the private network it is the way onto:\n${written}`); - // The module that provides the private network, not the requirement it answers: `networking` - // is the domain a module offers, and what caused a rule is the module itself. - assert.match(written, /# mesh-wireguard — the private network/, - `the rule does not name what caused it:\n${written}`); - - // Loaded, and the mesh still works: a declaration reaches the other machine, which it cannot if - // the overlay is severed. This is the assertion that matters — a rule file that looks right and - // a mesh that has stopped are exactly what this is guarding against. - assert.match(await must("anchor", `nft list table inet mesh`), /dport 51820/); - - await must("laptop", `rm -f /etc/mesh-still-works`); - await must("anchor", `printf %s '{"module":"stillworks","version":"1",` + - `"resources":[{"id":"marker","type":"file","path":"/etc/mesh-still-works",` + - `"content":"yes","mode":"0644"}]}' > /tmp/stillworks.json`); - await must("anchor", `docker cp /tmp/stillworks.json mesh-controller:/stillworks.json`); - await mesh("module add /stillworks.json"); - await mesh("assign laptop stillworks"); - await mesh("push laptop"); - - let arrived = false; - for (let i = 0; i < 20 && !arrived; i++) { - arrived = (await on("laptop", `test -f /etc/mesh-still-works`)).ok; - if (!arrived) await new Promise((r) => setTimeout(r, 3000)); - } - assert.ok(arrived, - "the hub applied its own rule set and the mesh stopped reaching the other machine"); - - // And the other machine still reaches the hub over the private network, which is what the - // opened port is for. - assert.ok((await on("laptop", `ping -c 1 -W 5 anchor.internal`)).ok, - "the private network is down after the hub filtered itself"); - - await mesh("unassign anchor hubfilter"); - await mesh("unassign laptop stillworks"); - await mesh("push"); -}); +// "The hub can be filtered without severing the mesh" lived here, with an inline filter module on +// the anchor. Since ADR 0088 the hub IS filtered on every mesh — the base filter closes it until a +// filter module derives the rules — so the credential test above assigns the catalogue's and +// asserts the hub's port is admitted, and every cross-machine test after it is the proof the mesh +// was not severed. Retired 2026-09-22. test("a container reaches another machine by the name the mesh gave it", { skip, timeout: 900_000, @@ -1375,6 +1329,23 @@ test("a container reaches another machine by the name the mesh gave it", { // Defends novox/hq ADR 0007: a name under a machine is that machine, without the mesh being // told each one. +/** The catalogue's resolver modules on the control plane, added once; dnsmasq speaks on the bus so + * it is issued once per machine. The mesh writes the resolver's data as a fact the module + * declares (/etc/mesh-resolver/nodes.conf); no module of the mesh's own writes it any more. */ +const resolverIssued = new Set(); +async function resolverModules(machines: string[]): Promise { + for (const name of ["dnsmasq", "resolved-split-dns"]) { + const manifest = catalogueModule(name, held); + await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); + await mesh(`module add /${name}.json`); + } + for (const machine of machines) { + if (resolverIssued.has(machine)) continue; + await mesh(`module issue dnsmasq --node ${machine}`); + resolverIssued.add(machine); + } +} + test("every name under a machine resolves to that machine", { skip, timeout: 900_000, }, async () => { @@ -1385,9 +1356,11 @@ test("every name under a machine resolves to that machine", { // // The mesh writes the data and runs no daemon: a resolver is third-party software, and the // mesh has no business choosing one. So what is checked here is the mesh's half — that the - // data is right, complete, and follows the machines. - await mesh("assign anchor mesh-resolver"); - await mesh("assign laptop mesh-resolver"); + // data is right, complete, and follows the machines. The data is a fact the catalogue's dnsmasq + // declares, so that module is what is assigned; what it runs is the next test's concern. + await resolverModules(["anchor", "laptop"]); + await mesh("assign anchor dnsmasq"); + await mesh("assign laptop dnsmasq"); await mesh("push"); await new Promise((r) => setTimeout(r, 15_000)); @@ -1416,7 +1389,7 @@ test("every name under a machine resolves to that machine", { // Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the // network, so unassigning the domain module alone leaves the machine on the network — pulled // back by its own requirement. The mesh was right and this test was wrong the first time. - await mesh("unassign laptop mesh-resolver"); + await mesh("unassign laptop dnsmasq"); await mesh("unassign laptop networking"); await mesh("push anchor"); await new Promise((r) => setTimeout(r, 15_000)); @@ -1428,14 +1401,13 @@ test("every name under a machine resolves to that machine", { `the machine that stayed lost its own name:\n${after}`); await mesh("assign laptop networking"); - await mesh("unassign anchor mesh-resolver"); + await mesh("unassign anchor dnsmasq"); await mesh("push"); await new Promise((r) => setTimeout(r, 15_000)); }); test("a service is reached by a name under the machine it runs on", { - skip: skip || (!moduleExamples ? "set MESH_LAB_MODULES to mesh-controller's examples/modules" : false), - timeout: 900_000, + skip, timeout: 900_000, }, async () => { // postgres.novox.internal, plex.ace.internal — the first label is the service and the rest is // the node, so anything under a node's name must resolve to that node. What routes it once it @@ -1447,12 +1419,7 @@ test("a service is reached by a name under the machine it runs on", { // /etc/resolv.conf. The two claim the same thing precisely so that assigning the wrong one is a // refusal rather than a fight over the file — and picking the wrong one here would have been // testing that fight. - for (const name of ["dnsmasq", "resolved-split-dns"]) { - const manifest = readFileSync(`${moduleExamples}/${name}.json`, "utf8"); - await must("anchor", `cat > /tmp/${name}.json <<'MANIFEST'\n${manifest}\nMANIFEST`); - await must("anchor", `docker cp /tmp/${name}.json mesh-controller:/${name}.json`); - await mesh(`module add /${name}.json`); - } + await resolverModules(["anchor", "laptop"]); // Both machines, because a node resolves from its own copy — the same rule as everything else // it holds. A mesh where one machine answers for all of them stops resolving when that machine @@ -1679,305 +1646,9 @@ test("a third-party workload is adopted, with the credential it already had", { // Running them needs their images stocked and two provisioners built, which is a separate and // larger job. This is the half that can be known now, and it is the half where a design fault // would live. -test("the real modules resolve together, and compose a declaration a host accepts", { - skip, timeout: 300_000, -}, async (t) => { - // Everything the catalogue holds, except two whose names this mesh is already running under: - // `registry` is the artifact store the suite stood up, and `umami` is the adopted workload — - // adding the catalogue's manifests would replace the records of modules that are live and - // assigned, and the adopted umami would suddenly require a database it never asked for. - const modules = ["postgres", "keycloak", "gitea", "minio", "mailu", - "redis", "grafana", "nextcloud", "searxng", "influxdb", "verdaccio"]; - const planned: string[] = []; - for (const name of modules) { - const raw = readFileSync( - `${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8"); - // Pointed at this scenario's registry before being added, exactly as the forge is. - // - // **Not cosmetic.** Some of these name an image the mesh builds, whose digest does not exist - // until it is built — so the file legitimately carries a placeholder, and composing a - // declaration from it is refused (novox/hq 04-ISSUES/025). Planning what could never run is - // what this test used to do. - const pinned = pinnedInto(raw, held); - // What this scenario does not serve cannot be redirected, and a module still naming a - // placeholder cannot be planned — the refusal is the point (novox/hq 04-ISSUES/025). Skipped - // and said, rather than silently dropped: a planning test quietly covering four modules - // instead of five is the false coverage this suite exists to prevent. - const left = stillUnpinned(pinned); - if (left.length > 0) { - console.log(`skipping ${name}: this scenario serves no ${left.join(", ")}`); - continue; - } - planned.push(name); - await must("anchor", `printf %s ${quote(pinned)} > /${name}.json`); - await must("anchor", `docker cp /${name}.json mesh-controller:/${name}.json`); - await mesh(`module add /${name}.json`); - } - - // **Put the machine back whatever happens.** Tests here share one mesh, so what this one - // assigns is what the next one inherits. Written at the end of the body once, it was skipped - // the first time this test failed — and the next test's push was refused by a module this one - // had left behind, which reads as a fault in the test that was actually working. - t.after(async () => { - for (const name of planned) await mesh(`unassign anchor ${name}`).catch(() => {}); - }); - - // Assigned one at a time, because assignment resolves the whole set and says so immediately. - // A refusal here is the graph rejecting something, which is the point of asking. - for (const name of planned) { - await mesh(`assign anchor ${name}`); - } - - const plan = await mesh("plan anchor --json", 120_000); - const declaration = JSON.parse(plan.slice(plan.indexOf("{"))); - const byId = new Map( - (declaration.resources as any[]).map((r) => [r.id, r])); - const ids = [...byId.keys()]; - - // Every module's own network, which only exists because more than one container needs to reach - // another by name. - for (const id of ["postgres.net", "keycloak.net", "minio.net", "mailu.net"]) { - assert.ok(byId.has(id), `${id} is missing; ${ids.length} resources: ${ids.join(", ")}`); - assert.equal(byId.get(id).type, "network"); - } - - // The cross-module edge: keycloak asked for a database and was told where it is and given a - // credential. Neither file is anything keycloak's manifest could have written. - const bound = [...byId.values()].find((r) => - r.type === "file" && r.path === "/var/lib/keycloak/database.json"); - assert.ok(bound, `keycloak was never told where its database is: ${ids.join(", ")}`); - assert.match(JSON.stringify(bound), /postgres/, - "keycloak's binding does not name what answered its requirement"); - - // The password, alone in a file and sealed. It is a password and nothing else, so nothing reads - // it as configuration — novox/hq 04-ISSUES/023 and the playbook both turn on that distinction. - const credential = [...byId.values()].find((r) => - r.type === "file" && r.path === "/var/lib/keycloak/database.secret"); - assert.ok(credential, `keycloak was given no credential for its database: ${ids.join(", ")}`); - assert.ok(credential.sealed, "keycloak's credential is not sealed, so the mesh can read it"); - assert.ok(!credential.content, "a credential arrived as content rather than sealed"); - - // And the connection itself, which keycloak could not have written: the address and port come - // from what the provider serves, and the user name from what the mesh decided both ends would - // call this consumer (novox/hq 04-ISSUES/023). - const connection = [...byId.values()].find((r) => - r.type === "file" && r.path === "/var/lib/keycloak/database.env"); - assert.ok(connection, "keycloak was given no database configuration"); - assert.match(connection.content, /KC_DB_USERNAME=mesh_[a-z0-9_]+_keycloak/, - `keycloak was not told what name to present:\n${connection.content}`); - assert.doesNotMatch(connection.content, /\$\{bound:/, - `a placeholder reached the machine as a value:\n${connection.content}`); - - // The password is the one hole left open, and the sealed value travels beside it. The mesh - // discarded the plaintext, so the host is the only thing that can close it. - assert.match(connection.content, /KC_DB_PASSWORD=\$\{secret:postgres-database\}/, - `the password was not left for the host to fill:\n${connection.content}`); - assert.ok(connection.secrets?.["postgres-database"], - "the sealed credential did not travel with the file that needs it"); - assert.doesNotMatch(JSON.stringify(connection.content), /postgres-database":"[A-Za-z0-9+/]{24,}/, - "the credential was written into the configuration in the clear"); - - // And the provider was told who asked, which is what its provisioner reconciles against. - const grants = [...byId.values()].find((r) => - r.type === "file" && String(r.path).startsWith("/var/lib/postgres/grants")); - assert.ok(grants, "postgres was never told which modules were granted a database"); - assert.match(JSON.stringify(grants), /keycloak|gitea/, - "the grants file names neither module that asked for a database"); - - // Secrets reach containers as files, never as environment in the declaration. - const containers = [...byId.values()].filter((r) => r.type === "container"); - assert.ok(containers.length >= 12, - `only ${containers.length} containers; mailu alone is nine`); - for (const c of containers) { - for (const [key, value] of Object.entries(c.env ?? {})) { - // **An absolute path is a reference to a secret, not a secret**, and naming one is the - // whole design: the mesh delivers a credential as a file and a module says where. - // - // Excluded because `/` is in the base64 alphabet, so any path of 24 characters or more - // matched — `MESH_BROKER_FILE=/var/lib/mesh/builder/broker` was reported as a credential - // the broker would see. A check that fires on the right shape for the wrong reason is - // worse than none: it is the one that gets suppressed, and then it is not there when it - // is right. - if (String(value).startsWith("/")) continue; - assert.doesNotMatch(String(value), /^[A-Za-z0-9+/]{24,}={0,2}$/, - `${c.name} carries something secret-shaped in env.${key}, which the broker would see`); - } - } - -}); - -// The first of the real module descriptions to actually run. -// -// **Everything before this stopped at composing a declaration.** That proves the control plane and -// the host agree, and proves nothing about whether the thing described works — which is how five -// modules sat pinned to images that did not exist, parsing and resolving perfectly -// (novox/hq 04-ISSUES/025). -// -// The forge is the one worth running first. It needs a database from another module, a password it -// did not choose, and a connection string it could not have written itself: the address and port -// come from what the database serves, and the user name from what the mesh decided both ends would -// call it (04-ISSUES/022 and 023). If any of that is wrong it cannot start, and nothing else in -// this file would notice. -test("the forge runs, on a database the mesh gave it", { skip, timeout: 900_000 }, async () => { - for (const name of ["postgres", "gitea"]) { - const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/${name}.json`, "utf8"); - // An image the mesh builds has no digest until it is built, and one it does not build belongs - // to whichever registry served it. Only the first is rewritten; the second is pulled. - const pinned = pinnedInto(raw, held); - assert.deepEqual(stillUnpinned(pinned), [], - `${name} still names an image nothing serves, so it could not start`); - await must("anchor", `printf %s ${quote(pinned)} > /run-${name}.json`); - await must("anchor", `docker cp /run-${name}.json mesh-controller:/run-${name}.json`); - await mesh(`module add /run-${name}.json`); - await mesh(`assign anchor ${name}`); - } - await mesh("push anchor", 300_000); - - // **What the machine says it did, before asking what it produced.** This test pushed and then - // waited for a database role, so when the containers were never created at all it reported "no - // login was created" — true, and silent about the reason. A push that was accepted and an apply - // that worked are different facts, and the second is the one this depends on. - // - // And waited for, because `push` sends without waiting. Reading `status` the instant it returns - // describes the apply *before* this one, which is how this test came to report a missing - // container while insisting the machine was fine. - await settled("anchor"); - const running = (await on("anchor", `docker ps -a --format '{{.Names}} {{.Status}}'`)).out; - // Named with what the mesh meant to send, not only with what the machine has. A container that - // is absent because the mesh never asked for it and one that is absent because the machine could - // not make it are the same sentence here and different faults entirely, and the plan is the only - // thing that tells them apart. - assert.match(running, /\bpostgres\b/, - `the database module was pushed and no container for it exists:\n${running}\n\n` + - `what the mesh would send anchor:\n${await mesh("plan anchor")}\n\n` + - `${(await on("anchor", `tail -30 /var/log/mesh-host.log`)).out}`); - - // The database first: until the provisioner has made the login, the forge has nothing to - // connect to and its own start would prove only that it retries. - const psql = async (q: string) => - (await on("anchor", - `docker exec postgres psql -U postgres -qAt -c ${quote(q)}`, 60_000)).out.trim(); - - // Both halves in one poll. The provisioner makes the role and then the database, and a test - // that waited for the first and checked the second once was racing the gap between two - // statements — it lost, once, eighteen seconds into a run. - let made = ""; - for (let i = 0; i < 40 && made !== "t"; i++) { - made = await psql("select true from pg_roles where rolname = 'mesh_anchor_gitea'" + - " and exists (select from pg_database where datname = 'gitea')"); - if (made !== "t") await new Promise((r) => setTimeout(r, 3000)); - } - assert.equal(made, "t", - `no login was created for the forge:\n${(await on("anchor", "docker logs mesh-provision-postgres 2>&1 | tail -20")).out}`); - - // And the forge itself, answering. Not that its container exists — that it serves. - // - // On the port the mesh assigned, not the one the module declared (novox/hq ADR 0038): the - // module says 3000 and the machine publishes wherever the mesh put it. Read from the plan, - // because the plan is the same composition a push sends. - const planned = await mesh("plan anchor --json", 120_000); - const mapping = (JSON.parse(planned.slice(planned.indexOf("{"))).resources as any[]) - .find((r) => r.id === "gitea.server")?.ports - ?.map(String).find((p: string) => p.endsWith(":3000")); - assert.ok(mapping, "the plan does not say where the machine publishes the forge"); - const at = mapping.split(":")[0]; - let answered = false; - let said = { out: "", ok: false }; - for (let i = 0; i < 60 && !answered; i++) { - said = await on("anchor", `curl -sf -o /dev/null -w '%{http_code}' --max-time 5 http://127.0.0.1:${at}/`, 30_000); - answered = said.out.trim().startsWith("2") || said.out.trim() === "303"; - if (!answered) await new Promise((r) => setTimeout(r, 5000)); - } - assert.ok(answered, - `the forge never answered (last: ${said.out.trim()}):\n` + - `${(await on("anchor", "docker logs gitea 2>&1 | tail -25")).out}`); - - // **The credential actually worked.** A forge that started and could not reach its database - // would still answer on its port, so the log is where the difference lives. - const log = (await on("anchor", "docker logs gitea 2>&1 | tail -60")).out; - assert.doesNotMatch(log, /password authentication failed|connection refused|does not exist/i, - `the forge started and could not use the database it was given:\n${log}`); - - await mesh("unassign anchor gitea"); - await mesh("unassign anchor postgres"); - await mesh("push anchor", 300_000); -}); - -test("a consumer's cache grant means exactly its own keys", { skip, timeout: 600_000 }, async (t) => { - // The third provision after a database and a bucket, and the first whose tenancy is enforced - // by the store's own ACL rather than by separate namespaces: every consumer shares one - // keyspace, so the grant is a pattern — and the test is that the pattern means what the - // manifest said, in both directions. - const raw = readFileSync(`${process.env["MESH_LAB_MODULES"]}/redis.json`, "utf8"); - const pinned = pinnedInto(raw, held); - assert.deepEqual(stillUnpinned(pinned), [], - "redis still names an image nothing serves, so it could not start"); - await must("anchor", `printf %s ${quote(pinned)} > /run-redis.json`); - await must("anchor", `docker cp /run-redis.json mesh-controller:/run-redis.json`); - await mesh("module add /run-redis.json"); - - // A consumer with no container: what is under test is the credential's reach, and files on the - // machine are enough to prove it — the same reduction the first credential test makes. - await must("anchor", `printf %s '{"module":"cachetest","version":"1",` + - `"requires":["redis-cache"],` + - `"contributes":{"redis-cache":{"prefix":"cachetest"}},` + - `"binds":{"redis-cache":"/var/lib/cachetest/cache.json"},` + - `"secrets":{"redis-cache":"/var/lib/cachetest/cache.secret"},` + - `"resources":[{"id":"state","type":"directory","path":"/var/lib/cachetest","mode":"0700"}]}' ` + - `> /cachetest.json`); - await must("anchor", `docker cp /cachetest.json mesh-controller:/cachetest.json`); - await mesh("module add /cachetest.json"); - - await mesh("assign anchor redis"); - await mesh("assign anchor cachetest"); - await mesh("push anchor", 300_000); - await settled("anchor"); - - t.after(async () => { - for (const name of ["cachetest", "redis"]) { - await mesh(`unassign anchor ${name}`).catch(() => {}); - } - await mesh("push anchor", 300_000).catch(() => {}); - }); - - // What the mesh told each end. The consumer's user name comes from its binding; the user's - // password from the sealed file beside it — both written by the host, neither invented here. - const bound = JSON.parse(await must("anchor", `cat /var/lib/cachetest/cache.json`)); - const user = bound.as; - assert.ok(user?.startsWith("mesh_"), `the binding does not carry a usable user: ${user}`); - const secret = (await must("anchor", `cat /var/lib/cachetest/cache.secret`)).trim(); - - // The provisioner has to have run before anything can authenticate. Waited for via the store - // itself: the user list, asked with the server's own password, which the conf file the host - // wrote holds on the machine. - const admin = (await must("anchor", - `awk '/^requirepass/ {print $2}' /var/lib/redis-module/redis.conf`)).trim(); - let granted = false; - for (let i = 0; i < 40 && !granted; i++) { - const users = (await on("anchor", - `docker exec redis redis-cli --no-auth-warning -a ${quote(admin)} ACL USERS`)).out; - granted = users.includes(user); - if (!granted) await new Promise((r) => setTimeout(r, 3000)); - } - assert.ok(granted, `no user was created for the consumer: -` + - `containers:\n${(await on("anchor", "docker ps -a --format '{{.Names}} {{.Status}}' | head -20")).out}\n` + - `the store:\n${(await on("anchor", "docker logs redis 2>&1 | tail -15")).out}\n` + - `the provisioner:\n${(await on("anchor", "docker logs mesh-provision-redis 2>&1 | tail -15")).out}`); - - const asConsumer = (command: string) => - on("anchor", `docker exec redis redis-cli --no-auth-warning ` + - `--user ${quote(user)} --pass ${quote(secret)} ${command}`); - - // Its own keys: usable. - assert.match((await asConsumer("SET cachetest:proof yes")).out, /OK/, - "the consumer cannot write under the prefix it was granted"); - assert.match((await asConsumer("GET cachetest:proof")).out, /yes/, - "the consumer cannot read back what it wrote"); - - // Anyone else's: refused by the store itself, which is the entire point of the grant. - assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i, - "the consumer wrote outside its prefix — the grant means more than the manifest said"); - assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i, - "the consumer can flush the store, which no tenant may"); -}); +// Three tests lived here that read the mesh's example modules, which moved to the catalogue: +// "the real modules resolve together" (whole-mesh-novox installs the catalogue's modules together +// and proves the composed declaration), "the forge runs, on a database the mesh gave it" (the same +// bed, gitea on the mesh's postgres) and "a consumer's cache grant means exactly its own keys" +// (mesh-grant-end-to-end, against the catalogue's redis). Retired 2026-09-22 rather than rewritten +// into copies of those beds (novox/hq issue 074). From 10cfbd094a27d6e981f98819b5549d94ce6e4466 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 01:27:52 +0200 Subject: [PATCH 3/5] Review: the cache grant's tenancy assertions move into the grant bed; retirement notes say what the beds prove; the builder binary is pushed on a warm return; the large bed needs the catalogue --- .../integration/mesh-grant-end-to-end.test.ts | 12 ++++++ test/integration/mesh.test.ts | 42 +++++++++++-------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/test/integration/mesh-grant-end-to-end.test.ts b/test/integration/mesh-grant-end-to-end.test.ts index 496d1c2..f546c23 100644 --- a/test/integration/mesh-grant-end-to-end.test.ts +++ b/test/integration/mesh-grant-end-to-end.test.ts @@ -227,4 +227,16 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a // writing the contributions rather than the bed. const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`); assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`); + + // A grant means exactly the consumer's own keys: under its name it reads and writes, outside it + // and on the server as a whole it is refused. Carried over from the large mesh bed's retired + // cache-grant test — without this a provisioner that granted everything would keep every bed green. + const asConsumer = (command: string) => + on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`); + assert.match((await asConsumer("SET cacheuser:proof yes")).out, /OK/, "the consumer cannot write under its own name"); + assert.match((await asConsumer("GET cacheuser:proof")).out, /yes/, "the consumer cannot read back what it wrote"); + assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i, + "the consumer wrote outside its own keys, so the grant means more than it says"); + assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i, + "the consumer flushed the whole server, so the grant means more than it says"); }); diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 245fe70..34b56df 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -24,7 +24,7 @@ import { raise } from "../../src/lifecycle/raise.ts"; import type { HeldImage } from "../../src/pinning.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, deriveTheFilterOn, catalogueIsPresent } from "./harness.ts"; import { incus } from "../../src/incus/client.ts"; import { machineName } from "../../src/lifecycle/names.ts"; import { ready, returnTo, keep, rememberStock, warmStock } from "../../src/warm.ts"; @@ -44,7 +44,8 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + // The anchor's filter and the resolvers are the catalogue's (ADR 0088, issue 074). + : catalogueIsPresent() || false; const SCENARIO = "two-nodes"; let instanceId = ""; @@ -202,6 +203,14 @@ function tokenFrom(said: string): string { /** The builder started by hand on the anchor, against the foundation broker's plain port on * loopback — the one that builds until a builder module can (see the retired test's note). */ async function startBuilder(): Promise { + // The binary is disk and survives a snapshot; a snapshot taken without it does not gain it on a + // return, so it is pushed whenever the machine has none. + if (!(await on("anchor", `test -x /usr/local/bin/mesh-builder`)).ok) { + await incus([ + "file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`, + "--mode", "0755", + ], 180_000); + } await must("anchor", `mkdir -p /var/lib/mesh-builder`); await must("anchor", `pgrep -x mesh-builder >/dev/null || ` + `(MESH_BROKER_AMQP='amqp://guest:guest@127.0.0.1:5672/' MESH_REGISTRY=${registry} ` + @@ -269,13 +278,7 @@ before(async () => { // A build machine, so anything here can ask the mesh to build something. Placed rather than // assumed: nothing else in this scenario would start one. - if (builder) { - await incus([ - "file", "push", builder, `${machineName(instanceId, "anchor")}/usr/local/bin/mesh-builder`, - "--mode", "0755", - ], 180_000); - await startBuilder(); - } + if (builder) await startBuilder(); if (warming) { // Snapshotted only now, with everything up: a state worth returning to is the one after the // part nobody wants to repeat. @@ -1386,9 +1389,9 @@ test("every name under a machine resolves to that machine", { // because a wildcard pointing at nothing resolves and then hangs — where an unresolvable name // fails at once and says which name it was. // - // Both, and that is not tidiness: `mesh-resolver` requires name resolution, which requires the - // network, so unassigning the domain module alone leaves the machine on the network — pulled - // back by its own requirement. The mesh was right and this test was wrong the first time. + // Both: the resolver's data follows the private network, so the machine leaves the network as + // well as the resolver, and what is asserted is that the machine that stayed is answered for and + // the one that left is not. await mesh("unassign laptop dnsmasq"); await mesh("unassign laptop networking"); await mesh("push anchor"); @@ -1646,9 +1649,12 @@ test("a third-party workload is adopted, with the credential it already had", { // Running them needs their images stocked and two provisioners built, which is a separate and // larger job. This is the half that can be known now, and it is the half where a design fault // would live. -// Three tests lived here that read the mesh's example modules, which moved to the catalogue: -// "the real modules resolve together" (whole-mesh-novox installs the catalogue's modules together -// and proves the composed declaration), "the forge runs, on a database the mesh gave it" (the same -// bed, gitea on the mesh's postgres) and "a consumer's cache grant means exactly its own keys" -// (mesh-grant-end-to-end, against the catalogue's redis). Retired 2026-09-22 rather than rewritten -// into copies of those beds (novox/hq issue 074). +// Three tests lived here that read the mesh's example modules, which moved to the catalogue. +// Retired 2026-09-22 rather than rewritten into copies of the beds that stand where they stood +// (novox/hq issue 074): "the real modules resolve together" — whole-mesh-novox installs the +// catalogue's modules together and its gate is the composed declaration accepted and every core +// container running; "the forge runs, on a database the mesh gave it" — the same bed, which gates +// on gitea running but does not yet ask it to answer on its port with the credential it was given, +// a gap that bed should close; "a consumer's cache grant means exactly its own keys" — its tenancy +// assertions (a write outside the consumer's keys and a FLUSHALL are refused) moved into +// mesh-grant-end-to-end, against the catalogue's redis. From 146d7da9ef817b4123f4b02cb153007c94ef4571 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 01:30:53 +0200 Subject: [PATCH 4/5] The large bed starts the hand-started builder where a build is asked for: it does not outlive the broker's first reconcile --- test/integration/mesh.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/mesh.test.ts b/test/integration/mesh.test.ts index 34b56df..632f623 100644 --- a/test/integration/mesh.test.ts +++ b/test/integration/mesh.test.ts @@ -1156,6 +1156,10 @@ test("a new commit reaches a machine that is already running the old one", { // novox/hq ADR 0010 names the real risk of replacing a pipeline with a comparison: losing the // question "did my change go out?". This is that question, end to end — a commit, a build, a // catalogue, and a machine that ends up running what the source says. + // The hand-started builder does not outlive a broker restart, and the foundation's broker is + // recreated when the first push reconciles it: a builder is (re)started here, where a build is + // asked for. The mesh's own builder is a module with a restart policy and needs none of this. + await startBuilder(); const repo = "/var/lib/mesh/builder/repositories/delivered"; const write = async (what: string) => await must("anchor", `mkdir -p ${repo} && printf %s '{"module":"delivered","version":"1",` + From 8a85e2d02e274f19762815db8f884cb41f88254e Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 01:41:16 +0200 Subject: [PATCH 5/5] The grant bed's tenancy assertions are scoped to the login's keyspace, as redis scopes the ACL --- test/integration/mesh-grant-end-to-end.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/integration/mesh-grant-end-to-end.test.ts b/test/integration/mesh-grant-end-to-end.test.ts index f546c23..fef1361 100644 --- a/test/integration/mesh-grant-end-to-end.test.ts +++ b/test/integration/mesh-grant-end-to-end.test.ts @@ -228,13 +228,14 @@ test("the mesh grants a consumer redis's cache, and the credential it delivers a const runtimeEnv = await must(`docker inspect mesh-redis --format '{{json .Config.Env}}'`); assert.doesNotMatch(runtimeEnv, /MESH_SEAL_KEY/, `a seal key was set after all — ADR 0048 is not what ran:\n${runtimeEnv}`); - // A grant means exactly the consumer's own keys: under its name it reads and writes, outside it - // and on the server as a whole it is refused. Carried over from the large mesh bed's retired - // cache-grant test — without this a provisioner that granted everything would keep every bed green. + // A grant means exactly the consumer's own keys — `:*`, the keyspace redis's provisioner + // scopes the ACL user to: under it the consumer reads and writes, outside it and on the server as + // a whole it is refused. Carried over from the large mesh bed's retired cache-grant test — + // without this a provisioner that granted everything would keep every bed green. const asConsumer = (command: string) => on(`docker exec redis redis-cli --user ${quote(as)} --pass ${quote(password)} --no-auth-warning ${command} 2>&1`); - assert.match((await asConsumer("SET cacheuser:proof yes")).out, /OK/, "the consumer cannot write under its own name"); - assert.match((await asConsumer("GET cacheuser:proof")).out, /yes/, "the consumer cannot read back what it wrote"); + assert.match((await asConsumer(`SET ${as}:proof yes`)).out, /OK/, "the consumer cannot write under its own login"); + assert.match((await asConsumer(`GET ${as}:proof`)).out, /yes/, "the consumer cannot read back what it wrote"); assert.match((await asConsumer("SET other:proof no")).out, /NOPERM|no permissions/i, "the consumer wrote outside its own keys, so the grant means more than it says"); assert.match((await asConsumer("FLUSHALL")).out, /NOPERM|no permissions/i,