From 36f2fd1c2c6b9eccf48dc33932f20c130ce025d8 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 22 Sep 2026 12:26:34 +0200 Subject: [PATCH] A cache consumer presents the login it was granted, checked over the catalogue; the two-node bed's baserow keeps its own cache and the bed asserts it answers (issue 081) --- scenarios/two-node-db.yml | 7 +- test/beds-read-the-catalogue.test.ts | 4 +- ...ache-consumers-present-their-login.test.ts | 56 +++++++++ test/integration/assigned-two-node-db.test.ts | 112 ++++++------------ 4 files changed, 94 insertions(+), 85 deletions(-) create mode 100644 test/cache-consumers-present-their-login.test.ts diff --git a/scenarios/two-node-db.yml b/scenarios/two-node-db.yml index 9295550..dd9c57c 100644 --- a/scenarios/two-node-db.yml +++ b/scenarios/two-node-db.yml @@ -3,7 +3,8 @@ # The app-postgres provider and the mesh's own foundation store both want host port 5432, so they # cannot share a machine — the collision that blocked this chain single-node. Here the foundation # (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain — -# postgres and redis PROVIDERS plus the baserow and letta CONSUMERS that require them. Both +# the baserow and letta CONSUMERS of the one store (baserow keeps its cache inside its own container, +# novox/hq 081). Both # machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor, # over the underlay both machines already share. Provider and consumers are co-located on laptop, so # no cross-node module comms and no overlay are needed — and the 5432-vs-foundation conflict is gone @@ -25,8 +26,7 @@ machines: inbound: allow memory: 4GiB cpus: 4 - # The whole DB-consumer chain: postgres + redis providers, each a server and a broker-bound - # runtime, plus the baserow and letta consumer services and their tools runtimes — a dozen + # The DB consumers: the baserow and letta services and their tools runtimes — a handful of # containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server). # At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says # so — and convergence would present as "the mesh hangs". Six gigabytes gives it room. @@ -49,7 +49,6 @@ images: # provisioner (ADR 0048). - mesh-runtime-postgres:development - mesh-runtime-lavinmq:development - - mesh-runtime-redis:development - mesh-runtime-baserow:development - mesh-runtime-letta:development diff --git a/test/beds-read-the-catalogue.test.ts b/test/beds-read-the-catalogue.test.ts index d1d7cfd..abe26f4 100644 --- a/test/beds-read-the-catalogue.test.ts +++ b/test/beds-read-the-catalogue.test.ts @@ -50,8 +50,8 @@ const STILL_CARRIED: Record = { "assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"], why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" }, "assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" }, - "assigned-two-node-db.test.ts": { modules: ["redis", "baserow", "letta"], - why: "DIFFERS: redis mints its own secret, baserow drops its route requirement, letta drops its ports" }, + "assigned-two-node-db.test.ts": { modules: ["baserow", "letta"], + why: "DIFFERS: baserow drops its route requirement, letta drops its ports" }, "lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"], why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" }, }; diff --git a/test/cache-consumers-present-their-login.test.ts b/test/cache-consumers-present-their-login.test.ts new file mode 100644 index 0000000..227c5dc --- /dev/null +++ b/test/cache-consumers-present-their-login.test.ts @@ -0,0 +1,56 @@ +/** + * **A module that takes a shared-cache grant presents the login it was granted** (novox/hq 081). + * + * The cache provider scopes each consumer to an ACL user of its own, confined to keys under its + * login (novox/hq 080). A consumer that hands its software the password and not the login logs in + * as the server's default user: the grant is honoured by the provider and ignored by the consumer, + * and nothing notices while the default user is open. The grant bed proves the provider's half + * against a consumer written to the contract; this holds every catalogue module that asks for the + * cache to its half — the login, as `${bound:redis-cache:as}`, somewhere it hands its software. + * + * What it cannot see is whether the software also keeps its keys under that login: that is the + * software's, and a module whose software cannot (fixed key or channel names in its code) does not + * take the shared cache at all — baserow runs its own, and n8n in its shipped mode needs none. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts"; + +const CACHE = "redis-cache"; + +/** Whether a manifest hands its software the login it is granted for the cache. */ +function presentsTheLogin(manifest: string): boolean { + return manifest.includes(`\${bound:${CACHE}:as}`); +} + +test("the check sees a module that hands its software the password and not the login", () => { + const passwordOnly = JSON.stringify({ module: "m", requires: [CACHE], resources: [ + { id: "env", type: "file", path: "/x", content: `HOST=\${bound:${CACHE}:at}\nPASSWORD=\${secret:${CACHE}}\n` }] }); + const withLogin = JSON.stringify({ module: "m", requires: [CACHE], resources: [ + { id: "env", type: "file", path: "/x", content: `USER=\${bound:${CACHE}:as}\nPASSWORD=\${secret:${CACHE}}\n` }] }); + assert.equal(presentsTheLogin(passwordOnly), false); + assert.equal(presentsTheLogin(withLogin), true); +}); + +test("every catalogue module that takes the shared cache presents the login it was granted", (t) => { + const absent = catalogueIsPresent(); + if (absent) { + t.skip(`cannot check — ${absent}`); + return; + } + const offences: string[] = []; + for (const d of readdirSync(catalogueDir(), { withFileTypes: true })) { + const file = resolve(catalogueDir(), d.name, "module.json"); + if (!d.isDirectory() || !existsSync(file)) continue; + const text = readFileSync(file, "utf8"); + const m = JSON.parse(text) as { requires?: string[] }; + if (!(m.requires ?? []).includes(CACHE)) continue; + if (!presentsTheLogin(text)) offences.push(d.name); + } + assert.deepEqual(offences, [], + `these take the shared cache and never hand their software the login (\${bound:${CACHE}:as}), so they ` + + `log in as the server's default user — present the login, or run a cache of their own:\n ${offences.join("\n ")}`); +}); diff --git a/test/integration/assigned-two-node-db.test.ts b/test/integration/assigned-two-node-db.test.ts index 8708cb3..c87c837 100644 --- a/test/integration/assigned-two-node-db.test.ts +++ b/test/integration/assigned-two-node-db.test.ts @@ -8,15 +8,14 @@ * * This bed proves that across two machines. `anchor` is the control-node: it raises the foundation * and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner. - * `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus - * a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on - * anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins - * authenticate against the store. redis stays co-located on laptop for baserow's cache. + * `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each + * consumer's database is minted on the store on anchor and reached over the overlay: their bindings + * name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is + * its own, inside its container (novox/hq 081). * - * The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim - * from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on - * the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the - * mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed + * The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim + * from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta + * wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed * ONCE; laptop converges once with every one up, and the two consumers are provisioned against the * database the provider on their own node gave them. * @@ -25,12 +24,11 @@ * MESH_LAB_HOST_BINARY=.../mesh-host * MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock * - * HELPER — stock the four runtimes into the local daemon before the run (some may already be there): + * HELPER — stock the three runtimes into the local daemon before the run (some may already be there): * scripts/build-module-runtime.sh postgres /tmp/postgres.tar - * scripts/build-module-runtime.sh redis /tmp/redis.tar * scripts/build-module-runtime.sh baserow /tmp/baserow.tar * scripts/build-module-runtime.sh letta /tmp/letta.tar - * The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest) + * The service images (postgres:17-alpine, baserow/baserow:latest, letta/letta:latest) * must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls * what it runs from the scenario's own registry by digest. */ @@ -205,64 +203,17 @@ test("consumers on a joined node get their databases from the one foundation sto // they land on changes. // ================================================================================================ - // --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider - // is handed the minted credential already unsealed by the host (ADR 0048). It carries - // the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ---- - const redisManifest = JSON.stringify({ - module: "redis", - version: "1", - provides: [{ name: "redis-cache", scope: "mesh" }], - serves: { "redis-cache": { port: 6379 } }, - emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"], - consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"], - receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" }, - grants: { "redis-cache": "/var/lib/redis-module/grants" }, - "own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" }, - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" }, - { id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" }, - { id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" }, - { - id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644", - content: "requirepass ${secret:default}\nappendonly no\ndir /data\n", - }, - { - id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host", - volumes: [ - "/services/redis/data:/data", - "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro", - ], - args: ["/etc/redis/redis.conf"], - }, - { - id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"), - network: "host", - volumes: [ - "/var/lib/mesh/redis/broker:/run/secrets/broker:ro", - "/var/lib/redis-module/grants:/var/lib/redis-module/grants", - "/var/lib/redis-module/default.secret:/run/secrets/default:ro", - ], - env: { - MESH_BROKER_FILE: "/run/secrets/broker", - MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json", - GRANTS: "/var/lib/redis-module/grants", - MESH_PROVISION_REDIS: "127.0.0.1:6379", - MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default", - }, - }, - ], - }); - - // --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both - // from the grants the mesh writes, plus a runtime that serves baserow's tools. -------------------- + // --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh + // writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when + // no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared + // cache's per-consumer grant cannot confine (novox/hq 081). -------------------------------------- const baserowManifest = JSON.stringify({ module: "baserow", version: "1", - requires: ["postgres-database", "redis-cache"], + requires: ["postgres-database"], contributes: { "postgres-database": { name: "baserow" } }, - binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" }, - secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" }, + binds: { "postgres-database": "/var/lib/baserow/database.json" }, + secrets: { "postgres-database": "/var/lib/baserow/database.secret" }, "own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" }, resources: [ { id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" }, @@ -273,8 +224,7 @@ test("consumers on a joined node get their databases from the one foundation sto content: "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" + "DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" + - "DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" + - "REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" + + "DATABASE_PASSWORD=${secret:postgres-database}\n" + "SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n", }, { id: "net", type: "network", name: "baserow" }, @@ -401,8 +351,7 @@ test("consumers on a joined node get their databases from the one foundation sto await mesh(`push anchor`, 600_000); await settled("anchor"); - // The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat). - await addIssueAssign("redis", redisManifest); + // The consumers ride laptop. await addIssueAssign("baserow", baserowManifest); await addIssueAssign("letta", lettaManifest); await mesh(`push ${NODE}`); @@ -429,7 +378,6 @@ test("consumers on a joined node get their databases from the one foundation sto // THE co-residence proof — every module's containers up and stable on the second node. // ================================================================================================ const expected = [ - "redis", "mesh-redis", "baserow", "mesh-baserow", "letta", "mesh-letta", ]; @@ -505,7 +453,7 @@ test("consumers on a joined node get their databases from the one foundation sto // reached from laptop over the shared segment) — named for the node that runs it and the module. // ================================================================================================ const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`); - for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) { + for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) { assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`); } @@ -537,14 +485,20 @@ test("consumers on a joined node get their databases from the one foundation sto assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`); } - // baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret, - // and both arrived on the second node (a live redis AUTH is left to the redis single-module bed - // and the open provider-seal-key work). - const redisBound = await waitForBinding("/var/lib/baserow/redis.json"); - assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`); - assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`); - const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim(); - assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered"); + // baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a + // password it makes itself, and the mesh grants nothing (novox/hq 081). Up means it answers on + // loopback — PONG, or the challenge for the password it holds — and baserow logged no refusal. + let cache = { out: "", ok: false }; + const untilCache = Date.now() + 180_000; + while (Date.now() < untilCache) { + cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`); + if (/PONG|NOAUTH/.test(cache.out)) break; + await new Promise((r) => setTimeout(r, 5000)); + } + assert.match(cache.out, /PONG|NOAUTH/, `baserow's own cache is not running inside its container:\n${cache.out}`); + const baserowLog = (await on(NODE, `docker logs baserow 2>&1 | tail -400`)).out; + assert.doesNotMatch(baserowLog, /WRONGPASS|NOPERM|NOAUTH|Error .*connecting to .*6379/i, + `baserow could not use its cache:\n${baserowLog.split("\n").filter((l) => /redis|6379|NOAUTH|WRONGPASS|NOPERM/i.test(l)).slice(-15).join("\n")}`); // Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it. async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {