novox/mesh-lab
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

A cache consumer presents the login it was granted, checked over the catalogue; the two-node bed's baserow keeps its own cache and the bed asserts it answers (issue 081)

Browse Source
This commit is contained in:
jschoubben
2026-09-22 12:26:34 +02:00
parent 7a5d670e4e
commit 36f2fd1c2c
4 changed files with 94 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/integration/assigned-two-node-db.test.ts
+33 -79
View File
@@ -8,15 +8,14 @@
*
* This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
* and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus
* a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on
* anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins
* authenticate against the store. redis stays co-located on laptop for baserow's cache.
* `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each
* consumer's database is minted on the store on anchor and reached over the overlay: their bindings
* name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is
* its own, inside its container (novox/hq 081).
*
* The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on
* the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the
* mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
* The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
* from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta
* wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
* ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
* database the provider on their own node gave them.
*
@@ -25,12 +24,11 @@
* MESH_LAB_HOST_BINARY=.../mesh-host
* MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
*
* HELPER — stock the four runtimes into the local daemon before the run (some may already be there):
* HELPER — stock the three runtimes into the local daemon before the run (some may already be there):
* scripts/build-module-runtime.sh postgres /tmp/postgres.tar
* scripts/build-module-runtime.sh redis /tmp/redis.tar
* scripts/build-module-runtime.sh baserow /tmp/baserow.tar
* scripts/build-module-runtime.sh letta /tmp/letta.tar
* The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest)
* The service images (postgres:17-alpine, baserow/baserow:latest, letta/letta:latest)
* must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
* what it runs from the scenario's own registry by digest.
*/
@@ -205,64 +203,17 @@ test("consumers on a joined node get their databases from the one foundation sto
// they land on changes.
// ================================================================================================
// --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider
// is handed the minted credential already unsealed by the host (ADR 0048). It carries
// the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ----
const redisManifest = JSON.stringify({
module: "redis",
version: "1",
provides: [{ name: "redis-cache", scope: "mesh" }],
serves: { "redis-cache": { port: 6379 } },
emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
grants: { "redis-cache": "/var/lib/redis-module/grants" },
"own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
{ id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
{ id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
{ id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
{
id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
},
{
id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
volumes: [
"/services/redis/data:/data",
"/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
],
args: ["/etc/redis/redis.conf"],
},
{
id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
network: "host",
volumes: [
"/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
"/var/lib/redis-module/grants:/var/lib/redis-module/grants",
"/var/lib/redis-module/default.secret:/run/secrets/default:ro",
],
env: {
MESH_BROKER_FILE: "/run/secrets/broker",
MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
GRANTS: "/var/lib/redis-module/grants",
MESH_PROVISION_REDIS: "127.0.0.1:6379",
MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
},
},
],
});
// --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both
// from the grants the mesh writes, plus a runtime that serves baserow's tools. --------------------
// --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh
// writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when
// no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared
// cache's per-consumer grant cannot confine (novox/hq 081). --------------------------------------
const baserowManifest = JSON.stringify({
module: "baserow",
version: "1",
requires: ["postgres-database", "redis-cache"],
requires: ["postgres-database"],
contributes: { "postgres-database": { name: "baserow" } },
binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" },
secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" },
binds: { "postgres-database": "/var/lib/baserow/database.json" },
secrets: { "postgres-database": "/var/lib/baserow/database.secret" },
"own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
resources: [
{ id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
@@ -273,8 +224,7 @@ test("consumers on a joined node get their databases from the one foundation sto
content:
"DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
"DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
"DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" +
"REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" +
"DATABASE_PASSWORD=${secret:postgres-database}\n" +
"SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
},
{ id: "net", type: "network", name: "baserow" },
@@ -401,8 +351,7 @@ test("consumers on a joined node get their databases from the one foundation sto
await mesh(`push anchor`, 600_000);
await settled("anchor");
// The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat).
await addIssueAssign("redis", redisManifest);
// The consumers ride laptop.
await addIssueAssign("baserow", baserowManifest);
await addIssueAssign("letta", lettaManifest);
await mesh(`push ${NODE}`);
@@ -429,7 +378,6 @@ test("consumers on a joined node get their databases from the one foundation sto
// THE co-residence proof — every module's containers up and stable on the second node.
// ================================================================================================
const expected = [
"redis", "mesh-redis",
"baserow", "mesh-baserow",
"letta", "mesh-letta",
];
@@ -505,7 +453,7 @@ test("consumers on a joined node get their databases from the one foundation sto
// reached from laptop over the shared segment) — named for the node that runs it and the module.
// ================================================================================================
const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) {
for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) {
assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
}
@@ -537,14 +485,20 @@ test("consumers on a joined node get their databases from the one foundation sto
assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
}
// baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret,
// and both arrived on the second node (a live redis AUTH is left to the redis single-module bed
// and the open provider-seal-key work).
const redisBound = await waitForBinding("/var/lib/baserow/redis.json");
assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`);
assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`);
const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim();
assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered");
// baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a
// password it makes itself, and the mesh grants nothing (novox/hq 081). Up means it answers on
// loopback — PONG, or the challenge for the password it holds — and baserow logged no refusal.
let cache = { out: "", ok: false };
const untilCache = Date.now() + 180_000;
while (Date.now() < untilCache) {
cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`);
if (/PONG|NOAUTH/.test(cache.out)) break;
await new Promise((r) => setTimeout(r, 5000));
}
assert.match(cache.out, /PONG|NOAUTH/, `baserow's own cache is not running inside its container:\n${cache.out}`);
const baserowLog = (await on(NODE, `docker logs baserow 2>&1 | tail -400`)).out;
assert.doesNotMatch(baserowLog, /WRONGPASS|NOPERM|NOAUTH|Error .*connecting to .*6379/i,
`baserow could not use its cache:\n${baserowLog.split("\n").filter((l) => /redis|6379|NOAUTH|WRONGPASS|NOPERM/i.test(l)).slice(-15).join("\n")}`);
// Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {