From 599d41eb420ae2558553c972d77ccde5f1b328f8 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 12:11:52 +0200 Subject: [PATCH 1/8] Beds for the seed file and for the foundation's filter The vault bed grows into a create-once file and pushes again; the genesis bed probes the machine from the workstation for the whole install and asserts the store's port never answers while the bus's does. --- test/integration/assigned-vault.test.ts | 30 +++++++++++++++++++ test/integration/one-node-mesh.test.ts | 40 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/test/integration/assigned-vault.test.ts b/test/integration/assigned-vault.test.ts index 1059194..b3eaf9a 100644 --- a/test/integration/assigned-vault.test.ts +++ b/test/integration/assigned-vault.test.ts @@ -433,3 +433,33 @@ test("the operator recovers a root secret with a key the mesh never held, from t assert.ok(listed.some((k) => k.module === "mesh-vault" && k.name === "broker"), JSON.stringify(served)); assert.ok(!JSON.stringify(served).includes(onDisk), "secret_export returned a plaintext value"); }); + +test("a seeded file is created once, and what a program grows in it survives the next push", { + skip, timeout: 600_000, +}, async () => { + // novox/hq ADR 0087, issue 035. A file that says create-once is written when absent and left + // alone when present — content, mode and owner — so an access list a program persists into is + // not restored to its seed behind the program's back on every reconcile. + const manifest = JSON.stringify({ + module: "seed-test", version: "1", + resources: [ + { id: "dir", type: "directory", path: "/var/lib/seed-test", mode: "0755" }, + { id: "acl", type: "file", path: "/var/lib/seed-test/acl.conf", mode: "0600", "create-once": true, + content: "user default on\n" }, + ], + }); + await must(`printf %s ${quote(manifest)} > /tmp/seed-test.json && docker cp /tmp/seed-test.json mesh-controller:/seed-test.json`); + await mesh("module add /seed-test.json"); + await mesh(`assign ${MACHINE} seed-test`); + await mesh(`push ${MACHINE}`); + await settled(); + assert.equal(await must(`cat /var/lib/seed-test/acl.conf`), "user default on\n"); + + // The program grows it. + await must(`printf 'user app-one on >secret\n' >> /var/lib/seed-test/acl.conf`); + // A second push: the mesh reconciles everything it declared, and leaves the seed alone. + await mesh(`push ${MACHINE}`); + await settled(); + assert.equal(await must(`cat /var/lib/seed-test/acl.conf`), "user default on\nuser app-one on >secret\n", + "the seed was restored and what the program wrote into it was wiped"); +}); diff --git a/test/integration/one-node-mesh.test.ts b/test/integration/one-node-mesh.test.ts index 83c1855..7fcfcaa 100644 --- a/test/integration/one-node-mesh.test.ts +++ b/test/integration/one-node-mesh.test.ts @@ -40,6 +40,7 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync, writeFileSync, appendFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; +import net from "node:net"; import { resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; @@ -247,6 +248,31 @@ async function mesh(command: string, timeoutMs?: number): Promise { } } +/** The machine's address on the lab's uplink bridge — the one the workstation can dial. */ +async function uplinkAddressOf(machine: string): Promise { + const name = await instanceNameOf(instanceId, machine); + const listed = (await incus(["list", name, "--format", "csv", "-c", "4"], 30_000)).stdout; + const addresses = listed.split(/[,\s]+/).map((a: string) => a.trim()).filter((a: string) => /^10\./.test(a)); + assert.ok(addresses.length > 0, `no uplink address for ${machine} in:\n${listed}`); + return addresses[0] as string; +} + +/** Try to open a TCP connection every two seconds to each port, and remember whether any attempt ever succeeded. */ +function probeFromOutside(address: string, ports: number[]): { stop(): void; seen(): Map } { + const seen = new Map(ports.map((p) => [p, false])); + const attempt = () => { + for (const port of ports) { + const socket = net.connect({ host: address, port, timeout: 1000 }); + socket.once("connect", () => { seen.set(port, true); socket.destroy(); }); + socket.once("timeout", () => socket.destroy()); + socket.once("error", () => socket.destroy()); + } + }; + attempt(); + const timer = setInterval(attempt, 2000); + return { stop: () => clearInterval(timer), seen: () => seen }; +} + /** Until the anchor reports the last declaration genesis pushed as applied and current. */ async function settledAfterGenesis(withinMs = 300_000): Promise { const deadline = Date.now() + withinMs; @@ -508,6 +534,11 @@ before(async () => { // nothing held: no image is pre-resolved, because none is here to resolve to. await step("R1", GENESIS, null, async () => { try { + // Probed from the workstation for the whole install (novox/hq ADR 0088, issue 054): the + // store's client port must never answer from outside the machine, while the bus's must + // come to — which is also what proves the probe reaches the machine at all. + const probe = probeFromOutside(await uplinkAddressOf(CONTROL), [5432, 5671]); + try { raised = await genesis({ instanceId, node: CONTROL, @@ -538,6 +569,15 @@ before(async () => { ...(binary ? { hostBinary: binary } : {}), log: (m) => console.log(m), }); + } finally { + probe.stop(); + } + const seen = probe.seen(); + assert.equal(seen.get(5432), false, + "the store's port answered from outside the machine during the install — the base filter did not hold (issue 054)"); + assert.equal(seen.get(5671), true, + "the bus never answered from outside during the install, so the probe proves nothing — is the uplink address right?"); + raised.report.push(` filtered 5432 never answered from outside during the install; 5671 did`); } catch (err) { throw new Error(`the installer never ran: ${(err as Error).message}`); } -- 2.54.0 From 97d71503ee10dde63a28c657dc1ebcb41aafe397 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 12:32:00 +0200 Subject: [PATCH 2/8] Three beds deliver the secrets they copy from the catalogue as files (issue 073) --- test/integration/assigned-catalogue-apps.test.ts | 7 +++---- test/integration/assigned-catalogue-small.test.ts | 6 ++++-- test/integration/assigned-model-usage.test.ts | 10 ++++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/integration/assigned-catalogue-apps.test.ts b/test/integration/assigned-catalogue-apps.test.ts index d163449..fd62e0e 100644 --- a/test/integration/assigned-catalogue-apps.test.ts +++ b/test/integration/assigned-catalogue-apps.test.ts @@ -220,14 +220,13 @@ test("the mesh assigns mongodb, unifi, marrytts and postgres to one node in one { id: "mesh-state", type: "directory", path: "/var/lib/mesh/mongodb", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/mongodb", mode: "0700" }, { id: "grants", type: "directory", path: "/var/lib/mongodb/grants", mode: "0700" }, - { id: "root-env", type: "file", path: "/var/lib/mongodb/root.env", mode: "0600", content: "MONGO_INITDB_ROOT_PASSWORD=${secret:root}\n" }, { id: "data", type: "directory", path: "/services/mongodb/db-data", mode: "0700" }, { id: "net", type: "network", name: "mongodb" }, { id: "server", type: "container", name: "mongo", image: pinned("mongo"), network: "mongodb", - env: { MONGO_INITDB_ROOT_USERNAME: "root" }, - "env-file": ["/var/lib/mongodb/root.env"], - volumes: ["/services/mongodb/db-data:/data/db"], + // The root password reaches mongo as a file (novox/hq ADR 0086), the shape the catalogue's manifest has. + env: { MONGO_INITDB_ROOT_USERNAME: "root", MONGO_INITDB_ROOT_PASSWORD_FILE: "/run/secrets/root" }, + volumes: ["/services/mongodb/db-data:/data/db", "/var/lib/mongodb/root.secret:/run/secrets/root:ro"], }, { id: "runtime", type: "container", name: "mesh-mongodb", image: pinned("mesh-runtime-mongodb"), diff --git a/test/integration/assigned-catalogue-small.test.ts b/test/integration/assigned-catalogue-small.test.ts index 819b711..a977a75 100644 --- a/test/integration/assigned-catalogue-small.test.ts +++ b/test/integration/assigned-catalogue-small.test.ts @@ -229,14 +229,16 @@ test("the mesh assigns postgres, redis, minio and plex to one node in one push, { id: "mesh-state", type: "directory", path: "/var/lib/mesh/minio", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/minio", mode: "0700" }, { id: "grants", type: "directory", path: "/var/lib/minio/grants", mode: "0700" }, - { id: "root-env", type: "file", path: "/var/lib/minio/root.env", mode: "0600", content: "MINIO_ROOT_USER=meshroot\nMINIO_ROOT_PASSWORD=${secret:root}\n" }, + // The root password reaches minio as a file (novox/hq ADR 0086), the shape the catalogue's manifest has. + { id: "root-env", type: "file", path: "/var/lib/minio/root.env", mode: "0600", content: "MINIO_ROOT_USER=meshroot\n" }, { id: "data", type: "directory", path: "/services/minio/data/data1-1", mode: "0700" }, { id: "net", type: "network", name: "minio" }, { id: "server", type: "container", name: "minio", image: pinned("minio/minio"), network: "minio", args: ["server", "/data", "--console-address", ":9001"], "env-file": ["/var/lib/minio/root.env"], - volumes: ["/services/minio/data/data1-1:/data"], + env: { MINIO_ROOT_PASSWORD_FILE: "/run/secrets/root" }, + volumes: ["/services/minio/data/data1-1:/data", "/var/lib/minio/root.secret:/run/secrets/root:ro"], }, { id: "runtime", type: "container", name: "mesh-minio", image: pinned("mesh-runtime-minio"), diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index c809e63..9928511 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -249,10 +249,12 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/model-usage", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/model-usage", mode: "0700" }, + // The connection string carries the password, so it reaches the runtime as a file the mesh + // templates (novox/hq ADR 0086), the shape the catalogue's manifest has. { - id: "db-env", type: "file", path: "/var/lib/model-usage/db.env", mode: "0600", + id: "database-url", type: "file", path: "/var/lib/model-usage/database.url", mode: "0600", content: - "DATABASE_URL=postgresql://${bound:postgres-database:as}:${secret:postgres-database}@" + + "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@" + "${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n", }, { @@ -261,9 +263,9 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot volumes: [ "/var/lib/mesh/model-usage/broker:/run/secrets/broker:ro", "/var/lib/model-usage:/run/state", + "/var/lib/model-usage/database.url:/run/secrets/database-url:ro", ], - env: { MESH_BROKER_FILE: "/run/secrets/broker" }, - "env-file": ["/var/lib/model-usage/db.env"], + env: { MESH_BROKER_FILE: "/run/secrets/broker", DATABASE_URL_FILE: "/run/secrets/database-url" }, }, ], }); -- 2.54.0 From b7316d40fe392806fcd56ccb25096a2821d5fc3b Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 12:55:35 +0200 Subject: [PATCH 3/8] The three beds' inline postgres reads its superuser from a file, as the catalogue's does --- test/integration/assigned-catalogue-apps.test.ts | 7 +++---- test/integration/assigned-catalogue-small.test.ts | 7 +++---- test/integration/assigned-model-usage.test.ts | 7 +++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/test/integration/assigned-catalogue-apps.test.ts b/test/integration/assigned-catalogue-apps.test.ts index fd62e0e..8391c5e 100644 --- a/test/integration/assigned-catalogue-apps.test.ts +++ b/test/integration/assigned-catalogue-apps.test.ts @@ -168,14 +168,13 @@ test("the mesh assigns mongodb, unifi, marrytts and postgres to one node in one { id: "mesh-state", type: "directory", path: "/var/lib/mesh/postgres", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/postgres", mode: "0700" }, { id: "grants", type: "directory", path: "/var/lib/postgres/grants", mode: "0700" }, - { id: "superuser-env", type: "file", path: "/var/lib/postgres/superuser.env", mode: "0600", content: "POSTGRES_PASSWORD=${secret:superuser}\n" }, { id: "data", type: "directory", path: "/services/postgres/db-data", mode: "0700" }, { id: "net", type: "network", name: "postgres" }, { id: "server", type: "container", name: "postgres", image: pinned("postgres"), network: "postgres", - env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres" }, - "env-file": ["/var/lib/postgres/superuser.env"], - volumes: ["/services/postgres/db-data:/var/lib/postgresql/data"], + // The superuser reaches postgres as a file (novox/hq ADR 0086), the shape the catalogue's manifest has. + env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres", POSTGRES_PASSWORD_FILE: "/run/secrets/superuser" }, + volumes: ["/services/postgres/db-data:/var/lib/postgresql/data", "/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro"], }, { id: "runtime", type: "container", name: "mesh-postgres", image: pinned("mesh-runtime-postgres"), diff --git a/test/integration/assigned-catalogue-small.test.ts b/test/integration/assigned-catalogue-small.test.ts index a977a75..c99e3cd 100644 --- a/test/integration/assigned-catalogue-small.test.ts +++ b/test/integration/assigned-catalogue-small.test.ts @@ -178,14 +178,13 @@ test("the mesh assigns postgres, redis, minio and plex to one node in one push, { id: "mesh-state", type: "directory", path: "/var/lib/mesh/postgres", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/postgres", mode: "0700" }, { id: "grants", type: "directory", path: "/var/lib/postgres/grants", mode: "0700" }, - { id: "superuser-env", type: "file", path: "/var/lib/postgres/superuser.env", mode: "0600", content: "POSTGRES_PASSWORD=${secret:superuser}\n" }, { id: "data", type: "directory", path: "/services/postgres/db-data", mode: "0700" }, { id: "net", type: "network", name: "postgres" }, { id: "server", type: "container", name: "postgres", image: pinned("postgres"), network: "postgres", - env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres" }, - "env-file": ["/var/lib/postgres/superuser.env"], - volumes: ["/services/postgres/db-data:/var/lib/postgresql/data"], + // The superuser reaches postgres as a file (novox/hq ADR 0086), the shape the catalogue's manifest has. + env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres", POSTGRES_PASSWORD_FILE: "/run/secrets/superuser" }, + volumes: ["/services/postgres/db-data:/var/lib/postgresql/data", "/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro"], }, { id: "runtime", type: "container", name: "mesh-postgres", image: pinned("mesh-runtime-postgres"), diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index 9928511..f3c8c47 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -202,15 +202,14 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot { id: "mesh-state", type: "directory", path: "/var/lib/mesh/postgres", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/postgres", mode: "0700" }, { id: "grants", type: "directory", path: "/var/lib/postgres/grants", mode: "0700" }, - { id: "superuser-env", type: "file", path: "/var/lib/postgres/superuser.env", mode: "0600", content: "POSTGRES_PASSWORD=${secret:superuser}\n" }, { id: "data", type: "directory", path: "/services/postgres/db-data", mode: "0700" }, { id: "net", type: "network", name: "postgres" }, { id: "server", type: "container", name: "postgres", image: pinned("postgres"), network: "postgres", - env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres" }, - "env-file": ["/var/lib/postgres/superuser.env"], + // The superuser reaches postgres as a file (novox/hq ADR 0086), the shape the catalogue's manifest has. + env: { POSTGRES_USER: "postgres", POSTGRES_DB: "postgres", POSTGRES_PASSWORD_FILE: "/run/secrets/superuser" }, ports: ["5432"], - volumes: ["/services/postgres/db-data:/var/lib/postgresql/data"], + volumes: ["/services/postgres/db-data:/var/lib/postgresql/data", "/var/lib/postgres/superuser.secret:/run/secrets/superuser:ro"], }, { id: "runtime", type: "container", name: "mesh-postgres", image: pinned("mesh-runtime-postgres"), -- 2.54.0 From 67a1f6a0124a7d5699378f34cda272fd335427ea Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 12:58:35 +0200 Subject: [PATCH 4/8] The harness pins minio from quay.io, as the catalogue does (docker.io denies anonymous pulls) --- test/integration/harness.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 4cc9f99..7c45b02 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -77,7 +77,7 @@ const UPSTREAM = new Map([ ["lscr.io/linuxserver/radarr", "lscr.io/linuxserver/radarr@sha256:119aaa4a4f7349bcd2a136c5373a0d7925b5479915c7dfe0c0ad352db2a6d438"], ["lscr.io/linuxserver/sonarr", "lscr.io/linuxserver/sonarr@sha256:c19aa4ecdf03d73e1d5c901da33744cb7eb4d921f89bafed1ca264601d7fa224"], ["lscr.io/linuxserver/unifi-controller", "lscr.io/linuxserver/unifi-controller@sha256:fcd5d8b13a77a588c79c1b49e5fc9ad08115aa3bb1a3576c589c64908a68845f"], - ["minio/minio", "minio/minio@sha256:aefec8a86702aff0b0dcfdd9284bd7ab7c5631cbf9be63275799e6edcb30dfa2"], + ["minio/minio", "quay.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e"], // docker.io denies anonymous pulls; the catalogue pins quay.io (mesh-catalog #29) ["mongo", "mongo@sha256:e3fa459b4f4b72f3257c67a23c145e250b8b5700f033860392c68539b998bbe3"], ["ollama/ollama", "ollama/ollama@sha256:32931b46719f673c05fdbaa81ccb26da18ea4a1c57590a754874ab28ba269eb2"], ["postgres", "postgres@sha256:7456ef82e5f5bc43d997f4781bbd7c0d6389bff397564649a356e206ba473aee"], -- 2.54.0 From ba49f571dca651f8cc6aa983a35b57eb2f3fa025 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 13:12:19 +0200 Subject: [PATCH 5/8] The model-usage bed says what the machine knows when the login fails --- test/integration/assigned-model-usage.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index f3c8c47..daed8e0 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -305,6 +305,14 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot assert.equal(bound.provision, "postgres-database", `model-usage was bound the wrong provision: ${bound.provision}`); const pw = (await must(NODE, `cat /var/lib/model-usage/database.secret`)).trim(); assert.ok(bound.as && pw, `model-usage's login or password was empty (as=${bound.as})`); + // What the machine can say when the login below fails — asked now, so the failure carries it. + const account = async () => [ + "--- provisioner (mesh-postgres):", (await on(NODE, `docker logs --tail 25 mesh-postgres 2>&1`)).out, + "--- runtime (mesh-model-usage):", (await on(NODE, `docker logs --tail 25 mesh-model-usage 2>&1`)).out, + "--- grants:", (await on(NODE, `ls -la /var/lib/postgres/grants/; cat /var/lib/postgres/grants/mesh.json 2>&1 | head -30`)).out, + "--- roles:", (await on(NODE, `docker exec postgres psql -U postgres -tAc "select rolname from pg_roles where rolname like 'mesh%'" 2>&1`)).out, + "--- host log:", (await on(NODE, `tail -40 /var/log/mesh-host.log 2>&1`)).out, + ].join("\n"); const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@postgres:5432/${bound.as}?sslmode=disable`; async function usageQuery(sql: string): Promise<{ out: string; ok: boolean }> { @@ -329,7 +337,7 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot // for it before emitting, so the consumer's upsert has a table to write (a reading that arrives // before the table would be logged and lost). const tableReady = await waitFor("select to_regclass('usage') is not null", /^t$/m); - assert.match(tableReady, /^t$/m, `the usage table was never created (the consumer did not migrate):\n${tableReady}`); + assert.match(tableReady, /^t$/m, `the usage table was never created (the consumer did not migrate):\n${tableReady}\n${await account()}`); // Inject a usage event into the mesh. model-usage is a PURE CONSUMER, so its own broker account has // no publish right (mesh-controller grants write to mesh.events only to a module that declares `emits`). -- 2.54.0 From af14f1c9097254ac7afd8ced46fd70a86caa90f2 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 13:18:23 +0200 Subject: [PATCH 6/8] The model-usage bed can be left standing, and its account names the filters --- test/integration/assigned-model-usage.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index daed8e0..eb3966e 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -177,6 +177,10 @@ before(async () => { }, { timeout: 1_800_000 }); after(async () => { + if (process.env["MESH_LAB_KEEP"]) { + console.log(`\nLEFT STANDING: ${instanceId} — not destroyed (MESH_LAB_KEEP).`); + return; + } if (instanceId) await destroy(instanceId); await destroyAll(`${SCENARIO}-`); }, { timeout: 600_000 }); @@ -312,6 +316,11 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot "--- grants:", (await on(NODE, `ls -la /var/lib/postgres/grants/; cat /var/lib/postgres/grants/mesh.json 2>&1 | head -30`)).out, "--- roles:", (await on(NODE, `docker exec postgres psql -U postgres -tAc "select rolname from pg_roles where rolname like 'mesh%'" 2>&1`)).out, "--- host log:", (await on(NODE, `tail -40 /var/log/mesh-host.log 2>&1`)).out, + "--- containers:", (await on(NODE, `docker ps -a --format '{{.Names}} {{.Status}}'`)).out, + "--- postgres server:", (await on(NODE, `docker logs --tail 15 postgres 2>&1; ls -la /var/lib/postgres/`)).out, + "--- laptop filter:", (await on(NODE, `nft list ruleset 2>&1 | head -60`)).out, + "--- laptop → broker from a container:", (await on(NODE, `docker run --rm --network postgres alpine sh -c 'nc -zvw3 192.0.2.10 5671' 2>&1`)).out, + "--- anchor filter counters:", (await on("anchor", `nft -a list table inet mesh 2>&1 | head -60`)).out, ].join("\n"); const conn = `postgresql://${bound.as}:${encodeURIComponent(pw)}@postgres:5432/${bound.as}?sslmode=disable`; -- 2.54.0 From e085e31f95e9cf94287c9c28a3b154ad12201865 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 13:30:26 +0200 Subject: [PATCH 7/8] A bed that raises the foundation from the bundle derives the anchor's filter before it relies on the hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base ruleset (ADR 0088) admits ssh, the bus and the registry and nothing else until the mesh derives one, and the mesh derives one only where the filter module is assigned — which genesis does and these beds did not. Without it the hub's WireGuard port stayed closed, no joined node's tunnel formed, and every module dialling the anchor by its overlay name timed out fetching the broker's certificate; the model-usage bed showed it as a login that failed for a role never made. --- test/integration/assigned-model-usage.test.ts | 5 +- test/integration/harness.ts | 57 ++++++++++++++++++- test/integration/lavinmq-bed.test.ts | 5 +- test/integration/whole-mesh-ace.test.ts | 5 +- 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index eb3966e..f3194b8 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -38,7 +38,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -286,6 +286,9 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot await mesh(`overlay place ${NODE} --site lab`); await mesh("assign anchor networking"); await mesh(`assign ${NODE} networking`); + // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, + // and what a bed raised from the bundle must do itself (ADR 0088; see the harness). + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); await addIssueAssign("postgres", postgresManifest); await addIssueAssign("model-usage", modelUsageManifest); diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 7c45b02..313a6b0 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -9,7 +9,8 @@ */ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; import { isReachable, pools, supportedDrivers } from "../../src/incus/client.ts"; import { destroy, list } from "../../src/lifecycle/operate.ts"; import { diagramFromLive } from "../../src/diagram/from-live.ts"; @@ -230,3 +231,57 @@ export async function assertUniversalInvariants( } } } + +// --- the packet filter, where a bed raises the foundation without genesis ------------------------ + +/** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */ +export const FILTER_MODULE = "nftables"; + +/** The catalogue's manifest for a module, under either spelling of MESH_LAB_CATALOG: the modules + * directory, or the checkout that holds it. */ +export function catalogueManifest(module: string): string { + const dir = process.env["MESH_LAB_CATALOG"] ?? ""; + for (const candidate of [resolve(dir, module, "module.json"), resolve(dir, "modules", module, "module.json")]) { + if (existsSync(candidate)) return candidate; + } + throw new Error(`no manifest for ${module} under MESH_LAB_CATALOG=${dir || "(unset)"}`); +} + +function shellQuote(s: string): string { + return `'${s.replaceAll("'", `'\\''`)}'`; +} + +/** + * The foundation is raised behind a base ruleset that admits ssh, the bus and the registry and + * nothing else, "until the mesh derives one" (novox/hq ADR 0088) — and the mesh derives one only + * where the packet-filter module is assigned, which genesis does on the control-node. A bed that + * raises the foundation from the bundle skips genesis, so it must do the same before it relies on + * an overlay hub there: the hub's port is derived from its endpoint, and until the derived ruleset + * lands no joined node's tunnel forms, and every module that dials the anchor by its overlay name + * times out fetching the broker's certificate — the failure this helper was written after. + * + * Registered, assigned, pushed, and then WAITED FOR: what the machine loaded must admit the hub's + * port, which the base ruleset cannot. Call it after the hub is placed, so there is a port to derive. + */ +export async function deriveTheFilterOn(o: { + machine: string; node: string; hubPort: number; + must: (machine: string, command: string, timeoutMs?: number) => Promise; + mesh: (command: string, timeoutMs?: number) => Promise; + on: (machine: string, command: string, timeoutMs?: number) => Promise<{ out: string; ok: boolean }>; +}): Promise { + const manifest = readFileSync(catalogueManifest(FILTER_MODULE), "utf8"); + await o.must(o.machine, + `printf %s ${shellQuote(manifest)} > /tmp/${FILTER_MODULE}.json && docker cp /tmp/${FILTER_MODULE}.json mesh-controller:/${FILTER_MODULE}.json`); + await o.mesh(`module add /${FILTER_MODULE}.json`); + await o.mesh(`assign ${o.node} ${FILTER_MODULE}`); + await o.mesh(`push ${o.node}`, 600_000); + const admits = new RegExp(`udp dport ${o.hubPort} accept`); + const deadline = Date.now() + 180_000; + let ruleset = ""; + while (Date.now() < deadline) { + ruleset = (await o.on(o.machine, `nft list table inet mesh 2>&1`)).out; + if (admits.test(ruleset)) return ruleset; + await new Promise((r) => setTimeout(r, 5_000)); + } + assert.fail(`${FILTER_MODULE} is assigned to ${o.node} and the ruleset it loaded does not admit the hub's udp/${o.hubPort}:\n${ruleset}`); +} diff --git a/test/integration/lavinmq-bed.test.ts b/test/integration/lavinmq-bed.test.ts index 97a68a7..c90f6a3 100644 --- a/test/integration/lavinmq-bed.test.ts +++ b/test/integration/lavinmq-bed.test.ts @@ -41,7 +41,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -301,6 +301,9 @@ test("the lavinmq provider and its consumer ride laptop while the foundation bro await mesh(`overlay place ${NODE} --site lab`); await mesh("assign anchor networking"); await mesh(`assign ${NODE} networking`); + // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, + // and what a bed raised from the bundle must do itself (ADR 0088; see the harness). + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); await addIssueAssign("lavinmq", lavinmqManifest); await addIssueAssign("amqp-ping", amqpPingManifest); diff --git a/test/integration/whole-mesh-ace.test.ts b/test/integration/whole-mesh-ace.test.ts index 44cca2d..a0ad542 100644 --- a/test/integration/whole-mesh-ace.test.ts +++ b/test/integration/whole-mesh-ace.test.ts @@ -31,7 +31,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -284,6 +284,9 @@ test("the whole ace service set resolves, installs and converges on one node in await mesh(`overlay place ${NODE} --site lab`); await mesh("assign anchor networking"); await mesh(`assign ${NODE} networking`); + // The anchor's derived filter, admitting the hub's port — what genesis does on the control-node, + // and what a bed raised from the bundle must do itself (ADR 0088; see the harness). + await deriveTheFilterOn({ machine: "anchor", node: "anchor", hubPort: 51820, must, mesh, on }); // Add / issue / assign, resiliently: a module the node cannot host is recorded and skipped so one // bad assignment cannot poison the whole-node push. -- 2.54.0 From b0e7cf96d1509b2470e0b937501d0057cb77062c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 13:43:42 +0200 Subject: [PATCH 8/8] The apps bed's mongodb names its secrets' owner, as the catalogue's does, and says what the server said when the consumer cannot reach it --- test/integration/assigned-catalogue-apps.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/assigned-catalogue-apps.test.ts b/test/integration/assigned-catalogue-apps.test.ts index 8391c5e..a264809 100644 --- a/test/integration/assigned-catalogue-apps.test.ts +++ b/test/integration/assigned-catalogue-apps.test.ts @@ -215,6 +215,8 @@ test("the mesh assigns mongodb, unifi, marrytts and postgres to one node in one receives: { "mongodb-database": "/var/lib/mongodb/grants/mesh.json" }, grants: { "mongodb-database": "/var/lib/mongodb/grants" }, "own-secrets": { root: "/var/lib/mongodb/root.secret", broker: "/var/lib/mesh/mongodb/broker" }, + // The image drops to its own user before it reads the password file (ADR 0086; as the catalogue's). + "secrets-owner": "999:999", resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/mongodb", mode: "0700" }, { id: "state", type: "directory", path: "/var/lib/mongodb", mode: "0700" }, @@ -413,7 +415,9 @@ test("the mesh assigns mongodb, unifi, marrytts and postgres to one node in one assert.doesNotMatch(mongoRes.out, /authentication failed/i, `mongodb delivered a credential that does not authenticate:\n${mongoRes.out}\n---runtime log---\n${(await on(`docker logs mesh-mongodb 2>&1 | tail -30`)).out}`); assert.match(mongoRes.out, /MONGO_OK/, - `the consumer could not use its granted database as ${mongoAs}:\n${mongoRes.out}`); + `the consumer could not use its granted database as ${mongoAs}:\n${mongoRes.out}\n---containers---\n` + + `${(await on(`docker ps -a --format '{{.Names}} {{.Status}}'`)).out}\n---mongo log---\n` + + `${(await on(`docker logs mongo 2>&1 | tail -15`)).out}`); // --- mongodb and unifi serve their tools over their scoped accounts ------------------------------- let served = ""; -- 2.54.0