Merge pull request 'A cache consumer must present its login; the two-node bed runs green again' (#49) from multiple-fixes into main

scenarios/two-node-db.yml

@@ -3,11 +3,11 @@
 # The app-postgres provider and the mesh's own foundation store both want host port 5432, so they
 # cannot share a machine — the collision that blocked this chain single-node. Here the foundation
 # (store, broker, control) lives on `anchor` and NOTHING else; `laptop` runs the whole chain —
-# postgres and redis PROVIDERS plus the baserow and letta CONSUMERS that require them. Both
+# the baserow and letta CONSUMERS of the one store (baserow keeps its cache inside its own container,
+# novox/hq 081). Both
 # machines sit on one shared segment and enrol into the one mesh; only enrolment crosses to anchor,
-# over the underlay both machines already share. Provider and consumers are co-located on laptop, so
-# no cross-node module comms and no overlay are needed — and the 5432-vs-foundation conflict is gone
-# because the foundation store is on the OTHER node.
+# over the underlay both machines already share. The consumers' databases are minted on the one
+# foundation store on anchor and reached over the overlay; laptop runs no provider of its own.
 scenario: two-node-db
 
 segments:
@@ -25,8 +25,7 @@ machines:
     inbound: allow
     memory: 4GiB
     cpus: 4
-  # The whole DB-consumer chain: postgres + redis providers, each a server and a broker-bound
-  # runtime, plus the baserow and letta consumer services and their tools runtimes — a dozen
+  # The DB consumers: the baserow and letta services and their tools runtimes — a handful of
   # containers, two of them memory-hungry app servers (the Baserow all-in-one and the Letta server).
   # At the 2GiB the two-nodes bed gives this machine it would thrash — its own anchor comment says
   # so — and convergence would present as "the mesh hangs". Six gigabytes gives it room.
@@ -49,7 +48,6 @@ images:
   # provisioner (ADR 0048).
   - mesh-runtime-postgres:development
   - mesh-runtime-lavinmq:development
-  - mesh-runtime-redis:development
   - mesh-runtime-baserow:development
   - mesh-runtime-letta:development
 

scenarios/whole-mesh-ace.yml

@@ -3,7 +3,7 @@
 # scenarios/whole-mesh-novox.yml; same topology, a different (larger, media-heavy) module set.
 #
 # Foundation (store, broker, control) rides `anchor` and NOTHING else; ALL of ace's services ride the
-# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
+# `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
 # providers co-located with them. The media stack (sonarr/radarr/lidarr/plex/bazarr/nzbget/
 # qbittorrent/bookshelf) shares the operator-owned library directories under /services/media (ADR
 # 0051 `accesses`); the test pre-creates them on the node, as the operator would, before the push —

test/beds-read-the-catalogue.test.ts

@@ -50,8 +50,8 @@ const STILL_CARRIED: Record<string, { modules: string[]; why: string }> = {
   "assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"],
     why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" },
   "assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" },
-  "assigned-two-node-db.test.ts": { modules: ["redis", "baserow", "letta"],
-    why: "DIFFERS: redis mints its own secret, baserow drops its route requirement, letta drops its ports" },
+  "assigned-two-node-db.test.ts": { modules: ["baserow", "letta"],
+    why: "DIFFERS: baserow drops its route requirement, letta drops its ports" },
   "lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"],
     why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" },
 };

test/cache-consumers-present-their-login.test.ts

@@ -0,0 +1,75 @@
+/**
+ * **A module that takes a shared-cache grant presents the login it was granted** (novox/hq 081).
+ *
+ * The cache provider scopes each consumer to an ACL user of its own, confined to keys under its
+ * login (novox/hq 080). A consumer that hands its software the password and not the login logs in
+ * as the server's default user: the grant is honoured by the provider and ignored by the consumer,
+ * and nothing notices while the default user is open. The grant bed proves the provider's half
+ * against a consumer written to the contract; this holds every catalogue module that asks for the
+ * cache to its half — the login, as `${bound:redis-cache:as}`, somewhere it hands its software.
+ *
+ * What it cannot see is whether the software also keeps its keys under that login: that is the
+ * software's, and a module whose software cannot (fixed key or channel names in its code) does not
+ * take the shared cache at all — baserow runs its own, and n8n in its shipped mode needs none.
+ */
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts";
+
+const CACHE = "redis-cache";
+
+/** What a module hands its software: every file it writes, and every container's environment and
+ *  arguments. A comment, a `why`, or a declared exception is not handed to anything. */
+function handedToSoftware(manifest: string): string[] {
+  const m = JSON.parse(manifest) as { resources?: Record<string, unknown>[] };
+  const out: string[] = [];
+  for (const r of m.resources ?? []) {
+    if (typeof r["content"] === "string") out.push(r["content"] as string);
+    if (r["env"] && typeof r["env"] === "object") out.push(...Object.values(r["env"] as Record<string, string>).map(String));
+    if (Array.isArray(r["args"])) out.push(...(r["args"] as unknown[]).map(String));
+  }
+  return out;
+}
+
+/** Whether a manifest hands its software the login it is granted for the cache. */
+function presentsTheLogin(manifest: string): boolean {
+  const login = `\${bound:${CACHE}:as}`;
+  return handedToSoftware(manifest).some((given) => given.includes(login));
+}
+
+test("the check sees a module that hands its software the password and not the login", () => {
+  const passwordOnly = JSON.stringify({ module: "m", requires: [CACHE], resources: [
+    { id: "env", type: "file", path: "/x", content: `HOST=\${bound:${CACHE}:at}\nPASSWORD=\${secret:${CACHE}}\n` }] });
+  const withLogin = JSON.stringify({ module: "m", requires: [CACHE], resources: [
+    { id: "env", type: "file", path: "/x", content: `USER=\${bound:${CACHE}:as}\nPASSWORD=\${secret:${CACHE}}\n` }] });
+  assert.equal(presentsTheLogin(passwordOnly), false);
+  assert.equal(presentsTheLogin(withLogin), true);
+  // Named only where no software reads it — a declared reason — is not presenting it.
+  const onlyInAReason = JSON.stringify({ module: "m", requires: [CACHE], resources: [
+    { id: "srv", type: "container", name: "s", image: "x", env: { PASSWORD: `\${secret:${CACHE}}` },
+      "secrets-in-environment": `the login is \${bound:${CACHE}:as}` }] });
+  assert.equal(presentsTheLogin(onlyInAReason), false);
+});
+
+test("every catalogue module that takes the shared cache presents the login it was granted", (t) => {
+  const absent = catalogueIsPresent();
+  if (absent) {
+    t.skip(`cannot check — ${absent}`);
+    return;
+  }
+  const offences: string[] = [];
+  for (const d of readdirSync(catalogueDir(), { withFileTypes: true })) {
+    const file = resolve(catalogueDir(), d.name, "module.json");
+    if (!d.isDirectory() || !existsSync(file)) continue;
+    const text = readFileSync(file, "utf8");
+    const m = JSON.parse(text) as { requires?: string[] };
+    if (!(m.requires ?? []).includes(CACHE)) continue;
+    if (!presentsTheLogin(text)) offences.push(d.name);
+  }
+  assert.deepEqual(offences, [],
+    `these take the shared cache and never hand their software the login (\${bound:${CACHE}:as}), so they ` +
+    `log in as the server's default user — present the login, or run a cache of their own:\n  ${offences.join("\n  ")}`);
+});

test/integration/assigned-two-node-db.test.ts

@@ -8,15 +8,14 @@
  *
  * This bed proves that across two machines. `anchor` is the control-node: it raises the foundation
  * and adopts `postgres` there, so `mesh-store` is the one store and `mesh-postgres` its provisioner.
- * `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database` — plus
- * a co-located `redis` (which holds no seat). Each consumer's database is minted on the store on
- * anchor and reached over the overlay: their bindings name `anchor.internal`, and their minted logins
- * authenticate against the store. redis stays co-located on laptop for baserow's cache.
+ * `laptop` joins and runs the CONSUMERS — baserow and letta, which require `postgres-database`. Each
+ * consumer's database is minted on the store on anchor and reached over the overlay: their bindings
+ * name `anchor.internal`, and their minted logins authenticate against the store. baserow's cache is
+ * its own, inside its container (novox/hq 081).
  *
- * The four manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
- * from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, redis runs on
- * the host network with a lab-local seal key, and baserow/letta wire their servers to the grant the
- * mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
+ * The three manifests are the committed catalogue shapes (novox/hq ADR 0039/0047/0048), verbatim
+ * from the catalogue-broad bed — postgres publishes 5432 so its consumers connect, and baserow/letta
+ * wire their servers to the grant the mesh writes. They are added, each issued a scoped broker account, assigned to laptop, and pushed
  * ONCE; laptop converges once with every one up, and the two consumers are provisioned against the
  * database the provider on their own node gave them.
  *
@@ -25,12 +24,11 @@
  *   MESH_LAB_HOST_BINARY=.../mesh-host
  *   MESH_LAB_BUNDLE=.../examples/foundation-first-node.lock
  *
- * HELPER — stock the four runtimes into the local daemon before the run (some may already be there):
+ * HELPER — stock the three runtimes into the local daemon before the run (some may already be there):
  *   scripts/build-module-runtime.sh postgres /tmp/postgres.tar
- *   scripts/build-module-runtime.sh redis    /tmp/redis.tar
  *   scripts/build-module-runtime.sh baserow  /tmp/baserow.tar
  *   scripts/build-module-runtime.sh letta    /tmp/letta.tar
- * The service images (postgres:17-alpine, redis:7-alpine, baserow/baserow:latest, letta/letta:latest)
+ * The service images (postgres, baserow, letta, each pinned by digest)
  * must be in the local daemon too; scenarios/two-node-db.yml stocks all of them, and each node pulls
  * what it runs from the scenario's own registry by digest.
  */
@@ -152,12 +150,13 @@ async function settled(node: string, withinMs = 1_200_000): Promise<void> {
     last = said;
     await new Promise((r) => setTimeout(r, 5000));
   }
-  // Timed out — capture what the node is actually doing so the failure is diagnosable.
-  const ps = (await on(NODE, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
-  const hostLog = (await on(NODE, `tail -80 /var/log/mesh-host.log`)).out;
+  // Timed out — capture what the node that did not answer is doing, so the failure is diagnosable.
+  // Its own machine, not the second node's: the anchor timing out used to print the laptop's log.
+  const ps = (await on(node, `docker ps -a --format '{{.Names}}\t{{.Status}}'`)).out;
+  const hostLog = (await on(node, `tail -80 /var/log/mesh-host.log`)).out;
   throw new Error(
     `${node} never caught up within ${Math.round(withinMs / 1000)}s.\nLast status:\n${last}\n` +
-      `--- ${NODE} docker ps -a ---\n${ps}\n--- ${NODE} mesh-host.log tail ---\n${hostLog}`);
+      `--- ${node} docker ps -a ---\n${ps}\n--- ${node} mesh-host.log tail ---\n${hostLog}`);
 }
 
 before(async () => {
@@ -205,82 +204,38 @@ test("consumers on a joined node get their databases from the one foundation sto
   // they land on changes.
   // ================================================================================================
 
-  // --- redis: a cache provider on the host network (127.0.0.1:6379). No seal key: the provider
-  // is handed the minted credential already unsealed by the host (ADR 0048). It carries
-  // the committed provides/serves/receives/grants so baserow's redis-cache requirement resolves. ----
-  const redisManifest = JSON.stringify({
-    module: "redis",
-    version: "1",
-    provides: [{ name: "redis-cache", scope: "mesh" }],
-    serves: { "redis-cache": { port: 6379 } },
-    emits: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    consumes: ["module.redis.cache.provisioned", "module.redis.cache.deprovisioned"],
-    receives: { "redis-cache": "/var/lib/redis-module/grants/mesh.json" },
-    grants: { "redis-cache": "/var/lib/redis-module/grants" },
-    "own-secrets": { default: "/var/lib/redis-module/default.secret", broker: "/var/lib/mesh/redis/broker" },
-    resources: [
-      { id: "mesh-state", type: "directory", path: "/var/lib/mesh/redis", mode: "0700" },
-      { id: "state", type: "directory", path: "/var/lib/redis-module", mode: "0700" },
-      { id: "grants-dir", type: "directory", path: "/var/lib/redis-module/grants", mode: "0700" },
-      { id: "data", type: "directory", path: "/services/redis/data", mode: "0700", owner: "999:999" },
-      {
-        id: "server-conf", type: "file", path: "/var/lib/redis-module/redis.conf", mode: "0644",
-        content: "requirepass ${secret:default}\nappendonly no\ndir /data\n",
-      },
-      {
-        id: "server", type: "container", name: "redis", image: pinned("redis"), network: "host",
-        volumes: [
-          "/services/redis/data:/data",
-          "/var/lib/redis-module/redis.conf:/etc/redis/redis.conf:ro",
-        ],
-        args: ["/etc/redis/redis.conf"],
-      },
-      {
-        id: "runtime", type: "container", name: "mesh-redis", image: pinned("mesh-runtime-redis"),
-        network: "host",
-        volumes: [
-          "/var/lib/mesh/redis/broker:/run/secrets/broker:ro",
-          "/var/lib/redis-module/grants:/var/lib/redis-module/grants",
-          "/var/lib/redis-module/default.secret:/run/secrets/default:ro",
-        ],
-        env: {
-          MESH_BROKER_FILE: "/run/secrets/broker",
-          MESH_RECEIVES: "/var/lib/redis-module/grants/mesh.json",
-          GRANTS: "/var/lib/redis-module/grants",
-          MESH_PROVISION_REDIS: "127.0.0.1:6379",
-          MESH_PROVISION_PASSWORD_FILE: "/run/secrets/default",
-        },
-      },
-    ],
-  });
-
-  // --- baserow: a consumer that requires postgres-database AND redis-cache; server wired to both
-  // from the grants the mesh writes, plus a runtime that serves baserow's tools. --------------------
+  // --- baserow: a consumer that requires postgres-database, its server wired to the grant the mesh
+  // writes, plus a runtime that serves baserow's tools. Its cache is its own — the image runs one when
+  // no REDIS_HOST is given — because baserow keeps keys and channels under fixed names a shared
+  // cache's per-consumer grant cannot confine (novox/hq 081). --------------------------------------
   const baserowManifest = JSON.stringify({
     module: "baserow",
     version: "1",
-    requires: ["postgres-database", "redis-cache"],
+    requires: ["postgres-database"],
     contributes: { "postgres-database": { name: "baserow" } },
-    binds: { "postgres-database": "/var/lib/baserow/database.json", "redis-cache": "/var/lib/baserow/redis.json" },
-    secrets: { "postgres-database": "/var/lib/baserow/database.secret", "redis-cache": "/var/lib/baserow/redis.secret" },
+    binds: { "postgres-database": "/var/lib/baserow/database.json" },
+    secrets: { "postgres-database": "/var/lib/baserow/database.secret" },
     "own-secrets": { "secret-key": "/var/lib/baserow/secret-key.secret", broker: "/var/lib/mesh/baserow/broker" },
     resources: [
       { id: "mesh-state", type: "directory", path: "/var/lib/mesh/baserow", mode: "0700" },
       { id: "state", type: "directory", path: "/var/lib/baserow", mode: "0700" },
-      { id: "data", type: "directory", path: "/services/baserow/data", mode: "0700", owner: "9999:9999" },
+      // 0755, as the image ships it: the cache it runs for itself does so as another user, who
+      // must be able to reach its own directory under this one (novox/hq 081).
+      { id: "data", type: "directory", path: "/services/baserow/data", mode: "0755", owner: "9999:9999" },
       {
         id: "server-env", type: "file", path: "/var/lib/baserow/server.env", mode: "0600",
         content:
           "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\n" +
           "DATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\n" +
-          "DATABASE_PASSWORD=${secret:postgres-database}\nREDIS_HOST=${bound:redis-cache:at}\n" +
-          "REDIS_PORT=${bound:redis-cache:port}\nREDIS_PROTOCOL=redis\nREDIS_PASSWORD=${secret:redis-cache}\n" +
+          "DATABASE_PASSWORD=${secret:postgres-database}\n" +
           "SECRET_KEY=${secret:secret-key}\nBASEROW_PUBLIC_URL=http://localhost\n",
       },
       { id: "net", type: "network", name: "baserow" },
       {
         id: "server", type: "container", name: "baserow", image: pinned("baserow/baserow"), network: "baserow",
         "env-file": ["/var/lib/baserow/server.env"],
+        // Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
+        "secrets-in-environment": "baserow reads DATABASE_PASSWORD and SECRET_KEY with os.getenv and has no _FILE twin (settings/base.py); not convertible",
         volumes: ["/services/baserow/data:/baserow/data"],
       },
       { id: "runtime-config", type: "file", path: "/var/lib/mesh/baserow/config.json", mode: "0600", content: "{}\n", merge: "json" },
@@ -325,6 +280,8 @@ test("consumers on a joined node get their databases from the one foundation sto
       {
         id: "server", type: "container", name: "letta", image: pinned("letta/letta"), network: "letta",
         "env-file": ["/var/lib/letta/server.env"],
+        // Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
+        "secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
       },
       { id: "runtime-config", type: "file", path: "/var/lib/mesh/letta/config.json", mode: "0600", content: "{}\n", merge: "json" },
       { id: "runtime-env", type: "file", path: "/var/lib/letta/runtime.env", mode: "0600", content: "MESH_LETTA_PASSWORD=${secret:server-password}\n" },
@@ -341,6 +298,8 @@ test("consumers on a joined node get their databases from the one foundation sto
           MESH_LETTA_CONFIG_FILE: "/run/config/config.json",
         },
         "env-file": ["/var/lib/letta/runtime.env"],
+        // Declared as the catalogue declares it (novox/hq ADR 0086, issue 041).
+        "secrets-in-environment": "the letta image is env-driven and its file-source support could not be verified; the mesh runtime can take its password from config.json (client.ts) \u2014 not yet converted",
         "restart-on": ["runtime-config"],
       },
     ],
@@ -401,8 +360,7 @@ test("consumers on a joined node get their databases from the one foundation sto
   await mesh(`push anchor`, 600_000);
   await settled("anchor");
 
-  // The consumers ride laptop; redis is an ordinary co-located provider (it holds no seat).
-  await addIssueAssign("redis", redisManifest);
+  // The consumers ride laptop.
   await addIssueAssign("baserow", baserowManifest);
   await addIssueAssign("letta", lettaManifest);
   await mesh(`push ${NODE}`);
@@ -429,7 +387,6 @@ test("consumers on a joined node get their databases from the one foundation sto
   // THE co-residence proof — every module's containers up and stable on the second node.
   // ================================================================================================
   const expected = [
-    "redis", "mesh-redis",
     "baserow", "mesh-baserow",
     "letta", "mesh-letta",
   ];
@@ -505,7 +462,7 @@ test("consumers on a joined node get their databases from the one foundation sto
   // reached from laptop over the shared segment) — named for the node that runs it and the module.
   // ================================================================================================
   const users = await must("anchor", `docker exec mesh-broker lavinmqctl list_users 2>&1`);
-  for (const acct of ["anchor-postgres", "laptop-redis", "laptop-baserow", "laptop-letta"]) {
+  for (const acct of ["anchor-postgres", "laptop-baserow", "laptop-letta"]) {
     assert.match(users, new RegExp(acct), `the scoped account ${acct} is not on the broker:\n${users}`);
   }
 
@@ -537,14 +494,26 @@ test("consumers on a joined node get their databases from the one foundation sto
     assert.match(pg.out, /^1$/m, `${mod} could not connect to its granted postgres database as ${bound.as}:\n${pg.out}`);
   }
 
-  // baserow also got its redis-cache binding: the mesh wrote the binding and unsealed the secret,
-  // and both arrived on the second node (a live redis AUTH is left to the redis single-module bed
-  // and the open provider-seal-key work).
-  const redisBound = await waitForBinding("/var/lib/baserow/redis.json");
-  assert.equal(redisBound.provision, "redis-cache", `baserow's redis binding is the wrong provision: ${redisBound.provision}`);
-  assert.ok(redisBound.as, `baserow's redis binding carries no login:\n${JSON.stringify(redisBound)}`);
-  const redisSecret = (await must(NODE, `cat /var/lib/baserow/redis.secret`)).trim();
-  assert.ok(redisSecret.length > 0, "baserow's redis secret was not delivered");
+  // baserow's cache is its own: with no REDIS_HOST the image runs one inside the container, under a
+  // password it makes itself, and the mesh grants nothing (novox/hq 081). Three things say so:
+  // baserow said it chose its own; the cache answers on loopback with the challenge for that password
+  // (PONG would be a cache anyone can use, and fails); and nothing was refused a login.
+  const baserowLog = async () => (await on(NODE, `docker logs baserow 2>&1`)).out;
+  let chose = "";
+  let cache = { out: "", ok: false };
+  const untilCache = Date.now() + 240_000;
+  while (Date.now() < untilCache) {
+    chose = await baserowLog();
+    cache = await on(NODE, `docker exec baserow redis-cli -h 127.0.0.1 ping 2>&1`);
+    if (/Using embedded baserow redis/.test(chose) && /NOAUTH/.test(cache.out)) break;
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  assert.match(chose, /Using embedded baserow redis/,
+    `baserow did not start its own cache:\n${chose.split("\n").filter((l) => /redis/i.test(l)).slice(-15).join("\n")}`);
+  assert.match(cache.out, /NOAUTH/,
+    `baserow's own cache does not answer with its password challenge inside the container:\n${cache.out}`);
+  assert.doesNotMatch(chose, /WRONGPASS|NOPERM/,
+    `baserow was refused by its cache:\n${chose.split("\n").filter((l) => /WRONGPASS|NOPERM/.test(l)).slice(-15).join("\n")}`);
 
   // Helper: wait for the mesh to write a consumer's bound file with an `as`, and parse it.
   async function waitForBinding(path: string): Promise<{ as: string; provision: string }> {

test/integration/whole-mesh-ace.test.ts

@@ -4,7 +4,7 @@
  * whole-mesh-novox.test.ts; same harness and topology, a larger, media-heavy module set.
  *
  * Foundation (store, broker, control) rides `anchor` and nothing else; ALL of ace's services ride the
- * `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres/redis
+ * `ace` node. An overlay is placed so the two DB consumers (baserow, letta) reach the postgres
  * providers co-located with them. The media stack shares the operator-owned library directories
  * under /services/media (ADR 0051 `accesses`): the mesh writes an `access` resource that CONFIRMS
  * each path exists and mounts it, but creates and chowns none of it — so before() pre-creates those
@@ -264,7 +264,7 @@ test("the whole ace service set resolves, installs and converges on one node in
 }, async () => {
   for (const d of DROPPED) console.log(`DROPPED ${d.name}: ${d.why}`);
 
-  // The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres/redis).
+  // The overlay, so a consumer's binding `at` is non-empty (baserow/letta reach postgres).
   await mesh("overlay place anchor --hub --endpoint 192.0.2.10:51820 --site lab");
   await mesh(`overlay place ${NODE} --site lab`);
   await mesh("assign anchor networking");