diff --git a/README.md b/README.md index e69ad53..a6f682b 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,10 @@ export MESH_LAB_ROUTE_PROXY=/route-proxy `MESH_LAB_HOST_BINARY` and `MESH_LAB_MODULES` do double duty: the repository each sits in is what `suite` rebuilds and what the receipt claims. Point the run at a repository and it is built and -claimed; leave it out and it is neither. +claimed; leave it out and it is neither. `MESH_LAB_CATALOG` is claimed without being built: a bed +installs a catalogue module by reading its manifest from that checkout when it runs, so the +receipt names the catalogue's commit too, and a run taken before a manifest changed says so +(novox/hq 04-ISSUES/073). Check before running a long suite — it says which of these are missing rather than skipping quietly: diff --git a/src/repos.ts b/src/repos.ts index c0330e0..4e9ce50 100644 --- a/src/repos.ts +++ b/src/repos.ts @@ -10,7 +10,7 @@ * pointed at is neither built nor claimed. */ -import { dirname } from "node:path"; +import { basename, dirname } from "node:path"; export interface Repositories { /** Absolute path to the repository root, by name. */ @@ -24,5 +24,16 @@ export function repositories(env: NodeJS.ProcessEnv = process.env): Repositories if (host) found["mesh-host"] = dirname(host); const modules = env["MESH_LAB_MODULES"]; if (modules) found["mesh-controller"] = dirname(dirname(modules)); + // The catalogue is read, not built: a bed installs a module by reading its manifest from this + // checkout at run time (novox/hq 04-ISSUES/073). A receipt that did not name the catalogue's + // commit could not say whether a catalogue change had been proven — the beds used to carry + // their own copies of the manifests, and then it could not. + const catalogue = env["MESH_LAB_CATALOG"]; + if (catalogue) found["mesh-catalog"] = catalogueRoot(catalogue); return found; } + +/** MESH_LAB_CATALOG is accepted under either spelling — the checkout, or its `modules` directory. */ +export function catalogueRoot(catalogue: string): string { + return basename(catalogue) === "modules" ? dirname(catalogue) : catalogue; +} diff --git a/test/beds-read-the-catalogue.test.ts b/test/beds-read-the-catalogue.test.ts new file mode 100644 index 0000000..fc1fb70 --- /dev/null +++ b/test/beds-read-the-catalogue.test.ts @@ -0,0 +1,107 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { catalogueIsPresent, catalogueDir } from "./integration/harness.ts"; + +/** + * A bed installs a catalogue module by reading the catalogue, never by carrying a copy. + * + * The beds used to build the manifests they install inline, as literals taken from the catalogue + * when each bed was written. The copies did not move when the catalogue did: six modules were + * converted to file-delivered secrets and not one bed ran the converted shape, because every bed + * ran its own copy (novox/hq 04-ISSUES/073). "Proven in the lab" then meant "the copy was proven". + * + * So: a manifest literal in a bed that names a catalogue module is refused, unless the bed is + * listed below with the reason it still carries one. The list is the debt, and it only shrinks. + * + * What this reads: `module: ""` and `"module": ""` with a `version` close by, either + * order, in test/integration/*.test.ts, against the catalogue's directory names. Skipped aloud + * where MESH_LAB_CATALOG is unset — a skip is reported, never silent. A bed that hid the name + * behind a computed string would pass — this is a fence, not a proof, and the reviewer of a bed + * that builds a manifest inline is the proof. + */ + +/** + * Beds that still carry an inline copy of a catalogue module's manifest, and why. Three reasons + * recur, and each names the work that removes the entry: + * + * BESIDE the catalogue's module CLAIMS the foundation's container (postgres claims mesh-store, + * lavinmq mesh-broker) and adopts it in place; the bed raises a second one beside the + * foundation's instead. Reading the catalogue changes what the bed raises — it would + * adopt — and the bed's assertions with it. + * WEARING the bed proves a mesh mechanism (a grant, a credential, a restart, a route) with a + * module cut down to the shape the mechanism needs — no upstream server, a secret in the + * environment, a requirement edge removed — and gives it a catalogue name. It is a mesh + * test wearing a catalogue module's name. It should carry a name of its own, or read the + * catalogue and meet the module's real requirements. + * DIFFERS a module bed whose copy differs from the catalogue in more than the lab may rewrite + * (an image, a port, an address). Reading the catalogue is the fix and needs a run. + */ +const STILL_CARRIED: Record = { + "assigned-catalogue-apps.test.ts": { modules: ["postgres", "mongodb", "unifi", "marrytts"], + why: "BESIDE (postgres); DIFFERS (unifi takes its credentials from the environment, mongodb and marrytts drop listens)" }, + "assigned-catalogue-media.test.ts": { modules: ["sonarr", "radarr"], + why: "DIFFERS: both drop the route requirement the catalogue declares, and take their API keys from the environment" }, + "assigned-catalogue-small.test.ts": { modules: ["postgres", "minio", "redis", "plex"], + why: "BESIDE (postgres); DIFFERS (minio's root password by env-file, redis minting its own secret instead of the vault's, plex without its server)" }, + "assigned-model-usage.test.ts": { modules: ["postgres"], why: "BESIDE" }, + "assigned-two-node-db.test.ts": { modules: ["redis", "baserow", "letta"], + why: "DIFFERS: redis mints its own secret, baserow drops its route requirement, letta drops its ports" }, + "lavinmq-bed.test.ts": { modules: ["lavinmq", "amqp-ping"], + why: "BESIDE (lavinmq, with a bootstrap step and a data directory the catalogue has not got); DIFFERS (amqp-ping names its entrypoint)" }, + "assigned-grafana.test.ts": { modules: ["grafana"], why: "WEARING: the sidecar alone, no Grafana, no route" }, + "assigned-plex.test.ts": { modules: ["plex"], why: "WEARING: the sidecar alone, no Plex, the token in the environment" }, + "assigned-redis.test.ts": { modules: ["redis"], why: "WEARING: its own secret, a lab seal key in the environment" }, + "assigned-sonarr.test.ts": { modules: ["sonarr"], why: "WEARING: the sidecar alone against a forged config.xml" }, + "mesh-grant-end-to-end.test.ts": { modules: ["redis"], why: "WEARING: a grant mechanism test" }, + "minio-grant-end-to-end.test.ts": { modules: ["minio"], why: "WEARING: a grant mechanism test, the root password by env-file" }, + "postgres-grant-end-to-end.test.ts": { modules: ["postgres"], why: "WEARING: a grant mechanism test, the superuser by env-file" }, + "provider-on-backend-network.test.ts": { modules: ["redis"], why: "WEARING: a network mechanism test" }, + "provider-uses-mesh-credential.test.ts": { modules: ["redis"], why: "WEARING: a credential mechanism test" }, + "runtime-restart-on-config.test.ts": { modules: ["grafana"], why: "WEARING: a restart mechanism test" }, + "route-forwarding.test.ts": { modules: ["route-proxy", "hello-web"], + why: "WEARING: route-proxy without its certificate authority, hello-web with the route shape ADR 0066 replaced" }, + "mesh.test.ts": { modules: ["postgres", "builder", "umami"], + why: "WEARING: a postgres with no resources, a builder that builds itself, an umami that is another module of that name" }, +}; + +const beds = resolve(import.meta.dirname, "integration"); + +test("a bed that installs a catalogue module reads the catalogue", (t) => { + const absent = catalogueIsPresent(); + if (absent) { + // Said, not silent: a check that cannot see the catalogue has checked nothing. + t.skip(`cannot check — ${absent}`); + return; + } + const names = new Set(readdirSync(catalogueDir(), { withFileTypes: true }) + .filter((d) => d.isDirectory() && existsSync(resolve(catalogueDir(), d.name, "module.json"))) + .map((d) => d.name)); + + const offences: string[] = []; + for (const file of readdirSync(beds).filter((f) => f.endsWith(".test.ts")).sort()) { + const text = readFileSync(resolve(beds, file), "utf8"); + const found = new Set(); + // A manifest literal: the module's name with its version close behind it. A `module:` key + // elsewhere (a table of what to register, a grant entry) has no version and is not one. + for (const m of text.matchAll(/(?:^|[\s{,])(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"[^}]{0,160}?(?:"version"|version)\s*:/g)) { + if (names.has(m[1]!)) found.add(m[1]!); + } + // And the other order — a literal that names its version first. + for (const m of text.matchAll(/(?:^|[\s{,])(?:"version"|version)\s*:\s*"[^"]*"[^}]{0,160}?(?:"module"|module)\s*:\s*"([a-z0-9.-]+)"/g)) { + if (names.has(m[1]!)) found.add(m[1]!); + } + const declared = STILL_CARRIED[file]; + for (const name of [...found].sort()) { + if (declared?.modules.includes(name)) continue; + offences.push(`${file}: an inline manifest for the catalogue's '${name}'`); + } + for (const name of declared?.modules ?? []) { + if (!found.has(name)) offences.push(`${file}: declared as still carrying '${name}', and it does not — remove the declaration`); + } + } + assert.deepEqual(offences, [], + `a bed carries a copy of a catalogue manifest; read it with catalogueModule() from the harness:\n ${offences.join("\n ")}`); +}); diff --git a/test/catalogue-module.test.ts b/test/catalogue-module.test.ts new file mode 100644 index 0000000..45195e3 --- /dev/null +++ b/test/catalogue-module.test.ts @@ -0,0 +1,87 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { catalogueModule } from "./integration/harness.ts"; +import type { HeldImage } from "../src/pinning.ts"; + +/** + * The shared loader thirteen beds install through (novox/hq 04-ISSUES/073, ADR 0089): it reads + * the catalogue's manifest and rewrites only what the lab must. Checked here against a catalogue + * written by the test, so the rules hold without a lab run. + */ + +const digest = (c: string) => "sha256:" + c.repeat(64); +const held: HeldImage[] = [ + { requested: "mesh-runtime-thing:development", repository: "mesh-runtime-thing", reference: digest("a") }, + { requested: "mesh-helper:development", repository: "mesh-helper", reference: digest("b") }, +]; + +function aCatalogueWith(manifest: object): () => void { + const root = mkdtempSync(join(tmpdir(), "mesh-lab-catalogue-")); + mkdirSync(join(root, "modules", "distribution"), { recursive: true }); + writeFileSync(join(root, "modules", "distribution", "module.json"), "{}"); + mkdirSync(join(root, "modules", "thing")); + writeFileSync(join(root, "modules", "thing", "module.json"), JSON.stringify(manifest)); + const before = process.env["MESH_LAB_CATALOG"]; + process.env["MESH_LAB_CATALOG"] = root; + return () => { + if (before === undefined) delete process.env["MESH_LAB_CATALOG"]; else process.env["MESH_LAB_CATALOG"] = before; + rmSync(root, { recursive: true, force: true }); + }; +} + +const thing = { + module: "thing", version: "1", + resources: [ + { id: "server", type: "container", name: "thing", image: "postgres@" + digest("c"), ports: ["8080", "9090:9090"], env: { A: "1" } }, + { id: "runtime", type: "container", name: "mesh-thing", artifact: "runtime" }, + ], + build: { artifacts: [{ name: "runtime", kind: "image", from: "Dockerfile" }] }, +}; + +test("the runtime artifact becomes the image the machine holds, and the build section goes", () => { + const restore = aCatalogueWith(thing); + try { + const m = JSON.parse(catalogueModule("thing", held)) as { build?: unknown; resources: Record[] }; + assert.equal(m.build, undefined); + const runtime = m.resources.find((r) => r["id"] === "runtime")!; + assert.equal(runtime["image"], digest("a")); + assert.equal(runtime["artifact"], undefined); + // An image already pinned to a digest passes through as written. + assert.equal(m.resources.find((r) => r["id"] === "server")!["image"], "postgres@" + digest("c")); + } finally { restore(); } +}); + +test("an artifact the bed did not name is refused, and a named one resolves to the stocked image", () => { + const helper = { ...thing, resources: [{ id: "helper", type: "container", name: "h", artifact: "helper" }] }; + const restore = aCatalogueWith(helper); + try { + assert.throws(() => catalogueModule("thing", held), /names the "helper" artifact/); + const m = JSON.parse(catalogueModule("thing", held, { artifacts: { helper: "mesh-helper" } })) as { resources: Record[] }; + assert.equal(m.resources[0]!["image"], digest("b")); + assert.throws(() => catalogueModule("thing", held, { artifacts: { helper: "mesh-nothing" } }), /stocked no such image/); + } finally { restore(); } +}); + +test("a host-port remap and a lab address are the only other things that change", () => { + const restore = aCatalogueWith(thing); + try { + const m = JSON.parse(catalogueModule("thing", held, { + ports: { "8080": "8090:8080" }, + env: { server: { B: "2" } }, + })) as { resources: Record[] }; + const server = m.resources.find((r) => r["id"] === "server")!; + assert.deepEqual(server["ports"], ["8090:8080", "9090:9090"]); + assert.deepEqual(server["env"], { A: "1", B: "2" }); + } finally { restore(); } +}); + +test("a manifest the catalogue does not have is refused by name", () => { + const restore = aCatalogueWith(thing); + try { + assert.throws(() => catalogueModule("nothing", held), /no manifest for nothing/); + } finally { restore(); } +}); diff --git a/test/integration/adopted-store-cross-node.test.ts b/test/integration/adopted-store-cross-node.test.ts index 4bae27f..25c83aa 100644 --- a/test/integration/adopted-store-cross-node.test.ts +++ b/test/integration/adopted-store-cross-node.test.ts @@ -21,29 +21,25 @@ */ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; -const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` : !binary || !existsSync(binary) ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle" - : false; + : catalogueIsPresent(); const SCENARIO = "adopted-store-cross-node"; const NODE = "node2"; -const catalogDir = process.env["MESH_LAB_CATALOG"] - ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") - ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + let instanceId = ""; let held: HeldImage[] = []; @@ -65,29 +61,12 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi async function mesh(command: string, timeoutMs?: number): Promise { return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); } -function pinned(reference: string): string { return onTheMachine(reference, held); } function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } -/** Load a committed module.json with its container images rewritten to the scenario's pinned digests. */ +/** The catalogue's manifest as the lab runs it (harness), and whether it needs a broker account. */ function loadManifest(name: string): { manifest: string; broker: boolean } { - const path = resolve(catalogDir, name, "module.json"); - const m = JSON.parse(readFileSync(path, "utf8")) as { - resources?: { type: string; image?: string; artifact?: string }[]; - }; - for (const r of m.resources ?? []) { - if (r.type !== "container") continue; - if (typeof r.image === "string") { - // A placeholder image (mesh-runtime-@0…0) resolves to the stocked digest, as redis does. - r.image = pinned(r.image); - } else if (typeof r.artifact === "string") { - // The bundle-model bed does not build, so resolve a module's runtime ARTIFACT to its stocked - // image directly — postgres/lavinmq name their provisioner by artifact, not a placeholder. - r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`); - delete r.artifact; - } - } - const manifest = JSON.stringify(m); - return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; + const manifest = catalogueModule(name, held); + return { manifest, broker: needsBrokerAccount(manifest) }; } function tokenFrom(said: string): string { const found = said.split("\n").map((l) => l.trim()).find((l) => l.length > 100 && !l.includes(" ")); @@ -105,7 +84,6 @@ async function install(name: string, node: string): Promise { before(async () => { if (skip) return; - assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`) }); instanceId = raised.instanceId; held = raised.images; diff --git a/test/integration/anthropic-bed.test.ts b/test/integration/anthropic-bed.test.ts index c9d3409..eb89404 100644 --- a/test/integration/anthropic-bed.test.ts +++ b/test/integration/anthropic-bed.test.ts @@ -49,7 +49,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -62,7 +62,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "anthropic-bed"; const MACHINE = "anchor"; @@ -194,8 +194,6 @@ after(async () => { test("model access refreshes on the manager node and delivers only the access token, never the refresh token", { skip, timeout: 1_500_000, }, async () => { - const managerImage = pinned("mesh-runtime-anthropic-manager"); - const consumerImage = pinned("mesh-runtime-anthropic-consumer"); // --- the licence, and the manager as its holder ------------------------------------------------ // The manager module is a HOLDER, named the licence's manager. It is delivered the refresh token; @@ -207,37 +205,15 @@ test("model access refreshes on the manager node and delivers only the access to await mesh(`licence use personal ${MACHINE} anthropic-manager`); // --- the manager module, deployed so the host delivers its bound facts -------------------------- - // Inline manifest mirroring the committed module.json: model-access holder, refresh token bound as a - // sealed secret, no node-key mount. The scheduled container installs as present state (ADR 0053); - // the test drives adopt/refresh directly for a deterministic flow rather than waiting on cron. - const managerManifest = JSON.stringify({ - module: "anthropic-manager", - version: "1", - requires: ["model-access"], - binds: { "model-access": "/var/lib/mesh/anthropic-manager/model.json" }, - secrets: { "model-access": "/var/lib/mesh/anthropic-manager/refresh-token" }, - "own-secrets": { broker: "/var/lib/mesh/anthropic-manager/broker" }, - emits: ["module.anthropic-manager.usage.read"], - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-manager", mode: "0700" }, - { id: "out", type: "directory", path: "/var/lib/mesh/anthropic-manager/out", mode: "0700" }, - { - id: "refresh", type: "container", name: "mesh-anthropic-manager-refresh", - image: managerImage, network: "host", schedule: "*/9 * * * *", - args: ["run", "/app/modules/anthropic-manager/dist/refresh/index.js"], - volumes: ["/var/lib/mesh/anthropic-manager:/run/state"], - env: { - MESH_ANTHROPIC_LICENCE: "personal", - MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token", - MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage", - MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/refresh-token", - MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", - MESH_ANTHROPIC_ACCESS_OUT: "/run/state/out/access-token", - MESH_ANTHROPIC_GRANT_OUT: "/run/state/out/grant.json", - MESH_ANTHROPIC_USAGE_OUT: "/run/state/out/usage.json", - }, - }, - ], + // The catalogue's own manifest (novox/hq 04-ISSUES/073): model-access holder, refresh token bound + // as a sealed secret, no node-key mount. The scheduled container installs as present state (ADR + // 0053); the test drives adopt/refresh directly for a deterministic flow rather than waiting on + // cron. The one lab rewrite: the OAuth endpoints point at the stub this bed raises below. + const managerManifest = catalogueModule("anthropic-manager", held, { + env: { refresh: { + MESH_ANTHROPIC_TOKEN_ENDPOINT: "http://127.0.0.1:9099/token", + MESH_ANTHROPIC_USAGE_ENDPOINT: "http://127.0.0.1:9099/usage", + } }, }); await must(`printf %s ${quote(managerManifest)} > /tmp/anthropic-manager.json && docker cp /tmp/anthropic-manager.json mesh-controller:/anthropic-manager.json`); await mesh(`module add /anthropic-manager.json`); @@ -329,32 +305,8 @@ test("model access refreshes on the manager node and delivers only the access to assert.match(submitted, /sealed to 1 holder/, submitted); // --- 5. deliver: deploy the consumer and push; it gets the sealed access token ------------------- - const consumerManifest = JSON.stringify({ - module: "anthropic-consumer", - version: "1", - requires: ["model-access"], - binds: { "model-access": "/var/lib/anthropic-consumer/model.json" }, - secrets: { "model-access": "/var/lib/anthropic-consumer/access-token" }, - "own-secrets": { broker: "/var/lib/mesh/anthropic-consumer/broker" }, - emits: ["module.anthropic-consumer.usage.session"], - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/anthropic-consumer", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/anthropic-consumer", mode: "0700" }, - { id: "claude-home", type: "directory", path: "/var/lib/anthropic-consumer/claude", mode: "0700" }, - { - id: "apply", type: "container", name: "mesh-anthropic-consumer-apply", - image: consumerImage, network: "host", schedule: "*/9 * * * *", - args: ["run", "/app/modules/anthropic-consumer/dist/apply/index.js"], - volumes: ["/var/lib/anthropic-consumer:/run/state"], - env: { - MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/access-token", - MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", - MESH_CLAUDE_CREDENTIALS_FILE: "/run/state/claude/.credentials.json", - MESH_CLAUDE_IDENTITY_FILE: "/run/state/claude/.claude.json", - }, - }, - ], - }); + // The catalogue's own manifest (novox/hq 04-ISSUES/073). + const consumerManifest = catalogueModule("anthropic-consumer", held); await must(`printf %s ${quote(consumerManifest)} > /tmp/anthropic-consumer.json && docker cp /tmp/anthropic-consumer.json mesh-controller:/anthropic-consumer.json`); await mesh(`module add /anthropic-consumer.json`); await mesh(`module issue anthropic-consumer --node ${MACHINE}`); diff --git a/test/integration/assigned-audit.test.ts b/test/integration/assigned-audit.test.ts index 22603fb..ebca781 100644 --- a/test/integration/assigned-audit.test.ts +++ b/test/integration/assigned-audit.test.ts @@ -22,7 +22,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -35,7 +35,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "audit-node"; const MACHINE = "anchor"; @@ -145,26 +145,10 @@ after(async () => { test("the mesh assigns the audit logger, and it consumes over the account the mesh delivered", { skip, timeout: 900_000, }, async () => { - // The assigned-module manifest (mesh-catalog), its runtime image the ID the machine holds. - const manifest = JSON.stringify({ - module: "audit-logger", - version: "1", - consumes: ["#"], - "own-secrets": { broker: "/var/lib/audit-logger/broker" }, - resources: [ - { id: "state", type: "directory", path: "/var/lib/audit-logger", mode: "0700" }, - { id: "trail", type: "directory", path: "/var/lib/audit-logger/trail", mode: "0700" }, - { - id: "run", type: "container", name: "mesh-audit-logger", image: pinned("mesh-runtime-audit"), - network: "host", - volumes: [ - "/var/lib/audit-logger/broker:/run/secrets/broker:ro", - "/var/lib/audit-logger/trail:/trail", - ], - env: { MESH_BROKER_FILE: "/run/secrets/broker", AUDIT_LOG: "/trail/audit.log" }, - }, - ], - }); + // The catalogue's manifest (novox/hq 04-ISSUES/073). Its runtime artifact is the image this + // scenario stocks under the module's slug, `mesh-runtime-audit` — built by scripts/build-runtime-image.sh + // before build-module-runtime.sh generalised it, and named as it was. + const manifest = catalogueModule("audit-logger", held, { artifacts: { runtime: "mesh-runtime-audit" } }); await must(`printf %s ${quote(manifest)} > /tmp/audit.json && docker cp /tmp/audit.json mesh-controller:/audit.json`); await mesh("module add /audit.json"); diff --git a/test/integration/assigned-catalogue-mqtt.test.ts b/test/integration/assigned-catalogue-mqtt.test.ts index 762738c..527a842 100644 --- a/test/integration/assigned-catalogue-mqtt.test.ts +++ b/test/integration/assigned-catalogue-mqtt.test.ts @@ -36,7 +36,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -49,7 +49,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "catalogue-mqtt"; const MACHINE = "anchor"; @@ -158,101 +158,11 @@ test("the mesh assigns mosquitto: a run-once step seeds dynsec before the broker skip, timeout: 1_500_000, }, async () => { // mosquitto's dynsec config: the plugin refuses to start unless dynamic-security.json holds an - // admin client, so the store MUST be seeded first. The `run-once` bootstrap container is declared - // BEFORE `server` (the broker) and reuses the module's runtime image; the host runs it to - // completion, then starts the broker. The runtime `server`/`runtime` shape mirrors the committed - // manifest, with images pinned to what this scenario serves by digest. - const mosquittoConf = - "persistence true\n" + - "persistence_location /mosquitto/data\n\n" + - "log_dest stdout\n" + - "log_type warning\n" + - "log_type error\n" + - "log_type notice\n\n" + - "# Every client authenticates; identities and their per-topic ACLs are managed\n" + - "# at runtime by the dynamic security plugin, whose store the plugin itself owns.\n" + - "allow_anonymous false\n" + - "plugin /usr/lib/mosquitto_dynamic_security.so\n" + - "plugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n" + - "# MQTT listener\n" + - "listener 1883\n\n" + - "# MQTT-over-WebSockets listener\n" + - "listener 8081\n" + - "protocol websockets\n"; - - const manifest = JSON.stringify({ - module: "mosquitto", - version: "1", - provides: [{ name: "mqtt-topic", scope: "mesh" }], - serves: { "mqtt-topic": {} }, - emits: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"], - // The events entrypoint subscribes to its own lifecycle events (an audit log), so it consumes - // them too — declared, or the foundation never makes the queue the runtime binds (ADR 0046). - consumes: ["module.mosquitto.topic.provisioned", "module.mosquitto.topic.deprovisioned"], - receives: { "mqtt-topic": "/var/lib/mosquitto-module/grants/mesh.json" }, - grants: { "mqtt-topic": "/var/lib/mosquitto-module/grants" }, - "own-secrets": { - admin: "/var/lib/mosquitto-module/admin.secret", - broker: "/var/lib/mesh/mosquitto/broker", - }, - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/mosquitto", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/mosquitto-module", mode: "0700" }, - { id: "grants-dir", type: "directory", path: "/var/lib/mosquitto-module/grants", mode: "0700" }, - // The broker runs as uid 1883, so the shared data directory it seeds into and persists to is - // its own. - { id: "data", type: "directory", path: "/services/mosquitto/data", mode: "0700", owner: "1883:1883" }, - { - id: "server-conf", type: "file", path: "/var/lib/mosquitto-module/mosquitto.conf", - mode: "0600", owner: "1883:1883", content: mosquittoConf, - }, - { id: "net", type: "network", name: "mosquitto" }, - // THE run-once step: seed dynsec offline, once, before the broker. It reuses the runtime image - // (`mesh-tools run ` imports mosquitto's bootstrap entrypoint, which writes the - // admin client into dynamic-security.json and chowns it to the broker's uid, then exits). It is - // declared BEFORE `server`; the host runs it to completion and requires exit 0 before starting - // the broker. - { - id: "bootstrap", type: "container", name: "mosquitto-bootstrap", - image: pinned("mesh-runtime-mosquitto"), "run-once": true, - volumes: [ - "/services/mosquitto/data:/mosquitto/data", - "/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro", - ], - env: { - MESH_PROVISION_MQTT: "mosquitto:1883", - MESH_PROVISION_ADMIN_USER: "mesh-admin", - MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin", - MESH_DYNSEC_FILE: "/mosquitto/data/dynamic-security.json", - }, - args: ["run", "/app/modules/mosquitto/dist/bootstrap/index.js"], - }, - { - id: "server", type: "container", name: "mosquitto", image: pinned("eclipse-mosquitto"), - network: "mosquitto", ports: ["1883", "8081"], - volumes: [ - "/services/mosquitto/data:/mosquitto/data", - "/var/lib/mosquitto-module/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro", - ], - }, - { - id: "runtime", type: "container", name: "mesh-mosquitto", - image: pinned("mesh-runtime-mosquitto"), network: "mosquitto", - volumes: [ - "/var/lib/mesh/mosquitto/broker:/run/secrets/broker:ro", - "/var/lib/mosquitto-module/grants:/var/lib/mosquitto-module/grants:ro", - "/var/lib/mosquitto-module/admin.secret:/run/secrets/admin:ro", - ], - env: { - MESH_BROKER_FILE: "/run/secrets/broker", - MESH_RECEIVES: "/var/lib/mosquitto-module/grants/mesh.json", - MESH_PROVISION_MQTT: "mosquitto:1883", - MESH_PROVISION_ADMIN_USER: "mesh-admin", - MESH_PROVISION_PASSWORD_FILE: "/run/secrets/admin", - }, - }, - ], - }); + // admin client, so the store MUST be seeded first. The catalogue's manifest declares a `run-once` + // bootstrap container BEFORE `server` (the broker), reusing the module's runtime image; the host + // runs it to completion, then starts the broker. The manifest is the catalogue's own, its runtime + // artifact the image this scenario stocked (novox/hq 04-ISSUES/073). + const manifest = catalogueModule("mosquitto", held); await must(`printf %s ${quote(manifest)} > /tmp/mosquitto.json && docker cp /tmp/mosquitto.json mesh-controller:/mosquitto.json`); await mesh("module add /mosquitto.json"); diff --git a/test/integration/assigned-model-usage.test.ts b/test/integration/assigned-model-usage.test.ts index f3194b8..fbea8f6 100644 --- a/test/integration/assigned-model-usage.test.ts +++ b/test/integration/assigned-model-usage.test.ts @@ -38,7 +38,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -51,7 +51,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "model-usage-bed"; /** The node that carries the postgres provider and the model-usage consumer. anchor carries only the @@ -190,7 +190,9 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot }, async () => { // ================================================================================================ // THE MANIFESTS — postgres verbatim from two-node-db (its server publishes 5432 so the consumer - // reaches it), and model-usage the committed catalogue shape with its images pinned. + // reaches it): a SECOND postgres beside the foundation's store, which the catalogue's postgres would + // instead claim and adopt in place. Still an inline copy, declared in beds-read-the-catalogue.test.ts + // (novox/hq 04-ISSUES/073). model-usage is the catalogue's. // ================================================================================================ const postgresManifest = JSON.stringify({ module: "postgres", @@ -233,45 +235,12 @@ test("usage events are upserted into model-usage's store — latest-per-key, bot ], }); - // --- model-usage: requires postgres-database, owns a provisioned store, consumes module.*.usage.*, - // runs a run-once migrate then the long-lived event consumer. Both containers on the host network so - // they reach the granted postgres (at the provider's address the mesh writes) and the broker. ------ - const modelUsageManifest = JSON.stringify({ - module: "model-usage", - version: "1", - // `mesh_laptop_model-usage` is 23 chars, over the 20 an S3 access key keeps (ADR 0049); a short - // slug makes the consumer identity `mesh_laptop_usage` (17). db/role/`as` all derive from it. - slug: "usage", - capabilities: ["container-runtime"], - requires: ["postgres-database"], - contributes: { "postgres-database": { name: "model_usage" } }, - binds: { "postgres-database": "/var/lib/model-usage/database.json" }, - secrets: { "postgres-database": "/var/lib/model-usage/database.secret" }, - consumes: ["module.*.usage.*"], - "own-secrets": { broker: "/var/lib/mesh/model-usage/broker" }, - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/model-usage", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/model-usage", mode: "0700" }, - // The connection string carries the password, so it reaches the runtime as a file the mesh - // templates (novox/hq ADR 0086), the shape the catalogue's manifest has. - { - id: "database-url", type: "file", path: "/var/lib/model-usage/database.url", mode: "0600", - content: - "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@" + - "${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n", - }, - { - id: "runtime", type: "container", name: "mesh-model-usage", - image: pinned("mesh-runtime-model-usage"), network: "host", - volumes: [ - "/var/lib/mesh/model-usage/broker:/run/secrets/broker:ro", - "/var/lib/model-usage:/run/state", - "/var/lib/model-usage/database.url:/run/secrets/database-url:ro", - ], - env: { MESH_BROKER_FILE: "/run/secrets/broker", DATABASE_URL_FILE: "/run/secrets/database-url" }, - }, - ], - }); + // --- model-usage: the catalogue's own manifest (novox/hq 04-ISSUES/073). It requires + // postgres-database, owns a provisioned store, consumes module.*.usage.*, and its runtime is on the + // host network so it reaches the granted postgres (at the provider's address the mesh writes) and + // the broker. Its slug keeps the consumer identity under the 20 characters an S3 access key allows + // (ADR 0049). ------------------------------------------------------------------------------------ + const modelUsageManifest = catalogueModule("model-usage", held); async function addIssueAssign(name: string, manifest: string): Promise { await must("anchor", `printf %s ${quote(manifest)} > /tmp/${name}.json && docker cp /tmp/${name}.json mesh-controller:/${name}.json`); diff --git a/test/integration/assigned-tools-confluence.test.ts b/test/integration/assigned-tools-confluence.test.ts index 00f3fed..faafcf1 100644 --- a/test/integration/assigned-tools-confluence.test.ts +++ b/test/integration/assigned-tools-confluence.test.ts @@ -29,7 +29,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -42,7 +42,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "tools-confluence"; const MACHINE = "anchor"; @@ -155,35 +155,9 @@ test("the mesh assigns confluence: its tools-only runtime comes up and serves th // confluence is tools-only and outbound-only: no service, no listener, no provisioner — just a // broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the // lab has no real Confluence, so the token points at nothing — and that is the case under test: the - // runtime must serve every tool regardless. The runtime container name and shape mirror the - // committed manifest, with the image pinned to what this scenario serves by digest. - const manifest = JSON.stringify({ - module: "confluence", - version: "1", - "own-secrets": { - token: "/var/lib/confluence/token", - broker: "/var/lib/mesh/confluence/broker", - }, - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/confluence", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/confluence", mode: "0700" }, - { id: "config", type: "file", path: "/var/lib/confluence/config.json", merge: "json", content: "{}", mode: "0600" }, - { - id: "runtime", type: "container", name: "mesh-runtime-confluence", - image: pinned("mesh-runtime-confluence"), network: "host", - volumes: [ - "/var/lib/confluence/config.json:/run/config/config.json:ro", - "/var/lib/confluence/token:/run/secrets/token:ro", - "/var/lib/mesh/confluence/broker:/run/secrets/broker:ro", - ], - env: { - MESH_CONFLUENCE_TOKEN_FILE: "/run/secrets/token", - MESH_CONFLUENCE_CONFIG_FILE: "/run/config/config.json", - MESH_BROKER_FILE: "/run/secrets/broker", - }, - }, - ], - }); + // runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime + // artifact the image this scenario stocked (novox/hq 04-ISSUES/073). + const manifest = catalogueModule("confluence", held); await must(`printf %s ${quote(manifest)} > /tmp/confluence.json && docker cp /tmp/confluence.json mesh-controller:/confluence.json`); await mesh("module add /confluence.json"); diff --git a/test/integration/assigned-tools-gitlab.test.ts b/test/integration/assigned-tools-gitlab.test.ts index 622882d..77b127f 100644 --- a/test/integration/assigned-tools-gitlab.test.ts +++ b/test/integration/assigned-tools-gitlab.test.ts @@ -28,7 +28,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -41,7 +41,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "tools-gitlab"; const MACHINE = "anchor"; @@ -154,35 +154,9 @@ test("the mesh assigns gitlab: its tools-only runtime comes up and serves the fu // gitlab is tools-only and outbound-only: no service, no listener, no provisioner — just a // broker-bound runtime serving the module's tools. Its own token is a mesh-minted own-secret; the // lab has no real GitLab, so the token points at nothing — and that is the case under test: the - // runtime must serve every tool regardless. The runtime container name and shape mirror the - // committed manifest, with the image pinned to what this scenario serves by digest. - const manifest = JSON.stringify({ - module: "gitlab", - version: "1", - "own-secrets": { - token: "/var/lib/gitlab/token", - broker: "/var/lib/mesh/gitlab/broker", - }, - resources: [ - { id: "mesh-state", type: "directory", path: "/var/lib/mesh/gitlab", mode: "0700" }, - { id: "state", type: "directory", path: "/var/lib/gitlab", mode: "0700" }, - { id: "config", type: "file", path: "/var/lib/gitlab/config.json", merge: "json", content: "{}", mode: "0600" }, - { - id: "runtime", type: "container", name: "mesh-runtime-gitlab", - image: pinned("mesh-runtime-gitlab"), network: "host", - volumes: [ - "/var/lib/gitlab/config.json:/run/config/config.json:ro", - "/var/lib/gitlab/token:/run/secrets/token:ro", - "/var/lib/mesh/gitlab/broker:/run/secrets/broker:ro", - ], - env: { - MESH_GITLAB_TOKEN_FILE: "/run/secrets/token", - MESH_GITLAB_CONFIG_FILE: "/run/config/config.json", - MESH_BROKER_FILE: "/run/secrets/broker", - }, - }, - ], - }); + // runtime must serve every tool regardless. The manifest is the catalogue's own, its runtime + // artifact the image this scenario stocked (novox/hq 04-ISSUES/073). + const manifest = catalogueModule("gitlab", held); await must(`printf %s ${quote(manifest)} > /tmp/gitlab.json && docker cp /tmp/gitlab.json mesh-controller:/gitlab.json`); await mesh("module add /gitlab.json"); diff --git a/test/integration/assigned-two-node-db.test.ts b/test/integration/assigned-two-node-db.test.ts index efe19d4..8708cb3 100644 --- a/test/integration/assigned-two-node-db.test.ts +++ b/test/integration/assigned-two-node-db.test.ts @@ -37,13 +37,12 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -56,16 +55,13 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "two-node-db"; /** The node that carries the whole DB-consumer chain. anchor carries only the foundation. */ const NODE = "laptop"; -const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; -const catalogDir = process.env["MESH_LAB_CATALOG"] - ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") - ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + let instanceId = ""; /** The mesh's own images, as the machines hold them. */ @@ -359,22 +355,11 @@ test("consumers on a joined node get their databases from the one foundation sto await mesh(`assign ${NODE} ${name}`); } - // Load a committed catalog module.json with its container images rewritten to this scenario's - // pinned digests, so the foundation store can be ADOPTED in place as the one postgres. + // The catalogue's manifest as the lab runs it (harness), so the foundation store can be ADOPTED + // in place as the one postgres. function loadManifest(name: string): { manifest: string; broker: boolean } { - const m = JSON.parse(readFileSync(resolve(catalogDir, name, "module.json"), "utf8")) as { - resources?: { type: string; image?: string; artifact?: string }[]; - }; - for (const r of m.resources ?? []) { - if (r.type !== "container") continue; - if (typeof r.image === "string") r.image = pinned(r.image); - else if (typeof r.artifact === "string") { - r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`); - delete r.artifact; - } - } - const manifest = JSON.stringify(m); - return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; + const manifest = catalogueModule(name, held); + return { manifest, broker: needsBrokerAccount(manifest) }; } async function installCatalog(name: string, node: string): Promise { const { manifest, broker } = loadManifest(name); diff --git a/test/integration/harness.ts b/test/integration/harness.ts index 313a6b0..a85422c 100644 --- a/test/integration/harness.ts +++ b/test/integration/harness.ts @@ -237,14 +237,106 @@ export async function assertUniversalInvariants( /** The packet filter genesis installs on the control-node, by the name the catalogue gives it. */ export const FILTER_MODULE = "nftables"; -/** The catalogue's manifest for a module, under either spelling of MESH_LAB_CATALOG: the modules - * directory, or the checkout that holds it. */ -export function catalogueManifest(module: string): string { - const dir = process.env["MESH_LAB_CATALOG"] ?? ""; - for (const candidate of [resolve(dir, module, "module.json"), resolve(dir, "modules", module, "module.json")]) { - if (existsSync(candidate)) return candidate; +// --- the catalogue: a bed installs a module by reading its manifest, never by carrying a copy ---- + +/** + * The catalogue's `modules/` directory: MESH_LAB_CATALOG under either spelling (the checkout, or + * its modules directory). Named, or absent — never guessed from a sibling path: the receipt claims + * the catalogue the run was pointed at (src/repos.ts), and a catalogue read from somewhere the + * receipt does not name is the drift this exists to close. + * + * Beds used to build the manifests they install inline, as literals copied from the catalogue when + * each bed was written. The copies did not move when the catalogue did, so a catalogue change was + * proven nowhere — and a bed that installs a copy proves the copy (novox/hq 04-ISSUES/073). A bed + * reads the catalogue, or it does not install a catalogue module; `beds-read-the-catalogue.test.ts` + * refuses an inline copy that names one. + */ +export function catalogueDir(): string { + const named = process.env["MESH_LAB_CATALOG"]; + if (!named) throw new Error("MESH_LAB_CATALOG is not set to a checkout of mesh-catalog (or its modules directory)"); + const candidates = [resolve(named, "modules"), resolve(named)]; + for (const dir of candidates) { + // Known by the registry's manifest, which genesis reads from the catalogue and always will — + // not the control plane's, which lives in the control plane's own repository (ADR 0069). + if (existsSync(resolve(dir, "distribution", "module.json"))) return dir; } - throw new Error(`no manifest for ${module} under MESH_LAB_CATALOG=${dir || "(unset)"}`); + throw new Error(`no catalogue: MESH_LAB_CATALOG=${named} and no distribution/module.json under ${candidates.join(" or ")}`); +} + +/** Whether a catalogue is where a bed will look — for a skip guard, which says so instead of failing. */ +export function catalogueIsPresent(): string | false { + try { catalogueDir(); return false; } catch (err) { return (err as Error).message; } +} + +/** The catalogue's manifest for a module, as a path. */ +export function catalogueManifest(module: string): string { + const path = resolve(catalogueDir(), module, "module.json"); + if (!existsSync(path)) throw new Error(`no manifest for ${module} at ${path}`); + return path; +} + +/** What the lab may rewrite in a catalogue manifest, and nothing else. */ +export interface ForTheLab { + /** + * The image repository each build artifact was built as on this workstation, by artifact name. + * A module's own runtime is `mesh-runtime-` by default — what `scripts/build-module-runtime.sh` + * tags and what the scenarios stock; a bed names it only where the scenario stocks another name. + * An artifact this does not name is refused: the bed must say what stands in for the builder. + */ + artifacts?: Record; + /** + * Host-port remaps, where one machine carries modules whose published ports collide — + * `{ "8080": "8090:8080" }`, applied to every container of the module. The container side never + * changes. + */ + ports?: Record | undefined; + /** + * Environment a container gets in the lab that it does not get in the mesh — an address the bed + * stands up in place of a real upstream, and nothing else. Merged over the manifest's own. + */ + env?: Record>; +} + +/** + * A catalogue manifest as a machine in the lab can run it: the mesh's build section gone (the lab + * stocks images rather than building), each artifact replaced by the image the machine holds for it, + * every image pinned to what the machine holds or the upstream digest the catalogue pins, and the + * declared lab rewrites applied. Everything else is the catalogue's, verbatim — which is the point. + */ +export function catalogueModule(module: string, held: HeldImage[], lab: ForTheLab = {}): string { + const m = JSON.parse(readFileSync(catalogueManifest(module), "utf8")) as { + resources?: { id: string; type: string; image?: string; artifact?: string; ports?: string[]; env?: Record }[]; + build?: unknown; + }; + const artifacts: Record = { runtime: `mesh-runtime-${module}`, ...(lab.artifacts ?? {}) }; + for (const r of m.resources ?? []) { + if (r.type !== "container") continue; + if (typeof r.artifact === "string") { + const repository = artifacts[r.artifact]; + assert.ok(repository, + `${module}'s container '${r.id}' names the "${r.artifact}" artifact, which the mesh would ` + + `build. The lab does not build: the bed must say which stocked image stands in for it ` + + `(artifacts: { ${r.artifact}: "" }).`); + const reference = referenceFor(held, repository); + assert.ok(reference, + `${module}'s "${r.artifact}" artifact is ${repository} and this scenario stocked no such ` + + `image. Add it to the scenario's images: and build it (scripts/build-module-runtime.sh ${module}).`); + r.image = reference; + delete r.artifact; + } else if (typeof r.image === "string") { + r.image = onTheMachine(r.image, held); + } + if (lab.ports && Array.isArray(r.ports)) r.ports = r.ports.map((p) => lab.ports![p] ?? p); + const env = lab.env?.[r.id]; + if (env) r.env = { ...(r.env ?? {}), ...env }; + } + delete m.build; + return JSON.stringify(m); +} + +/** Whether a manifest's runtime dials the broker — the module then needs a scoped broker account. */ +export function needsBrokerAccount(manifest: string): boolean { + return manifest.includes("MESH_BROKER_FILE"); } function shellQuote(s: string): string { diff --git a/test/integration/local-model-bed.test.ts b/test/integration/local-model-bed.test.ts index 7a01fe0..7263ff9 100644 --- a/test/integration/local-model-bed.test.ts +++ b/test/integration/local-model-bed.test.ts @@ -26,7 +26,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -39,7 +39,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "local-model-bed"; const MACHINE = "anchor"; @@ -153,47 +153,13 @@ after(async () => { test("a node hosting a model answers model-access, and the consumer is handed its endpoint", { skip, timeout: 1_500_000, }, async () => { - const ollamaImage = pinned("ollama/ollama"); - - // The provider: ollama runs the model server and `provides: ["model-access"]` at node scope, serving - // its port and model. It mints nothing — provides/serves are declaration the mesh reads, so there is - // no runtime container, only the server. - const ollamaManifest = JSON.stringify({ - module: "ollama", - version: "1", - capabilities: ["container-runtime"], - provides: [{ name: "model-access", scope: "node" }], - listens: [{ port: 11434, protocol: "tcp", from: "machine", why: "local consumers reaching the model server" }], - serves: { "model-access": { port: 11434, model: "llama3.2" } }, - resources: [ - { id: "state", type: "directory", path: "/services/ollama", mode: "0700" }, - { - id: "server", type: "container", name: "ollama", - image: ollamaImage, network: "host", env: { OLLAMA_HOST: "0.0.0.0:11434" }, - volumes: ["/services/ollama:/root/.ollama"], - }, - ], - }); - - // The consumer: it requires model-access and is answered by the local node. No secret (the local - // server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes. - const consumerManifest = JSON.stringify({ - module: "local-model-consumer", - version: "1", - slug: "local", - requires: ["model-access"], - binds: { "model-access": "/var/lib/local-model-consumer/model.json" }, - resources: [ - { id: "state", type: "directory", path: "/var/lib/local-model-consumer", mode: "0700" }, - { id: "config", type: "directory", path: "/var/lib/local-model-consumer/config", mode: "0700" }, - { - id: "openai-env", type: "file", path: "/var/lib/local-model-consumer/config/openai.env", mode: "0600", - content: - "OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1\n" + - "OPENAI_MODEL=${bound:model-access:model}\nOPENAI_API_KEY=local\n", - }, - ], - }); + // Both manifests are the catalogue's own (novox/hq 04-ISSUES/073). The provider: ollama runs the + // model server and `provides: ["model-access"]` at node scope, serving its port and model. It mints + // nothing — provides/serves are declaration the mesh reads, so there is no runtime container, only + // the server. The consumer requires model-access and is answered by the local node: no secret (the + // local server is keyless), only the bound endpoint facts, templated into an openai.env the mesh writes. + const ollamaManifest = catalogueModule("ollama", held); + const consumerManifest = catalogueModule("local-model-consumer", held); await addAssign("ollama", ollamaManifest); await addAssign("local-model-consumer", consumerManifest); diff --git a/test/integration/openai-bed.test.ts b/test/integration/openai-bed.test.ts index f80f327..885640c 100644 --- a/test/integration/openai-bed.test.ts +++ b/test/integration/openai-bed.test.ts @@ -24,7 +24,7 @@ import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueModule, catalogueIsPresent } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); @@ -37,7 +37,7 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "openai-bed"; const MACHINE = "anchor"; @@ -156,7 +156,6 @@ after(async () => { test("a static-key model-access licence delivers the operator's API key to the consumer, unchanged", { skip, timeout: 1_500_000, }, async () => { - const consumerImage = pinned("mesh-runtime-openai-consumer"); // --- the licence, a record with vendor openai (static-key) ------------------------------------- // No manager: a static-key licence has none (mesh-controller refuses `licence manager` on it). The @@ -172,33 +171,11 @@ test("a static-key model-access licence delivers the operator's API key to the c await mesh(`licence key personal --file /openai-key`); // --- deploy the consumer ----------------------------------------------------------------------- - // Inline manifest mirroring the committed module.json: a model-access holder whose delivered key + // The catalogue's own manifest (novox/hq 04-ISSUES/073): a model-access holder whose delivered key // arrives at its secret path. The scheduled apply container installs as present state (ADR 0053) and // its image is pulled at apply (schedule-pull); the test drives apply directly for a deterministic // flow rather than waiting on cron. - const consumerManifest = JSON.stringify({ - module: "openai-consumer", - version: "1", - requires: ["model-access"], - binds: { "model-access": "/var/lib/openai-consumer/model.json" }, - secrets: { "model-access": "/var/lib/openai-consumer/api-key" }, - resources: [ - { id: "state", type: "directory", path: "/var/lib/openai-consumer", mode: "0700" }, - { id: "config", type: "directory", path: "/var/lib/openai-consumer/config", mode: "0700" }, - { - id: "apply", type: "container", name: "mesh-openai-consumer-apply", - image: consumerImage, network: "host", schedule: "*/5 * * * *", - args: ["run", "/app/modules/openai-consumer/dist/apply/index.js"], - volumes: ["/var/lib/openai-consumer:/run/state"], - env: { - MESH_MODEL_ACCESS_SECRET_FILE: "/run/state/api-key", - MESH_MODEL_ACCESS_BIND_FILE: "/run/state/model.json", - MESH_OPENAI_ENV_FILE: "/run/state/config/openai.env", - MESH_OPENAI_CREDENTIALS_FILE: "/run/state/config/auth.json", - }, - }, - ], - }); + const consumerManifest = catalogueModule("openai-consumer", held); await must(`printf %s ${quote(consumerManifest)} > /tmp/openai-consumer.json && docker cp /tmp/openai-consumer.json mesh-controller:/openai-consumer.json`); await mesh(`module add /openai-consumer.json`); await mesh(`module issue openai-consumer --node ${MACHINE}`); diff --git a/test/integration/whole-mesh-ace.test.ts b/test/integration/whole-mesh-ace.test.ts index a0ad542..818c40d 100644 --- a/test/integration/whole-mesh-ace.test.ts +++ b/test/integration/whole-mesh-ace.test.ts @@ -25,19 +25,17 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine, deriveTheFilterOn } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, deriveTheFilterOn, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; -const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` @@ -45,14 +43,12 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "whole-mesh-ace"; const NODE = "ace"; -const catalogDir = process.env["MESH_LAB_CATALOG"] - ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") - ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + /** The operator-owned media library the ADR-0051 `accesses` point at — pre-created before the push. */ const MEDIA_DIRS = [ @@ -175,27 +171,17 @@ async function mesh(command: string, timeoutMs?: number): Promise { } /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ -function pinned(reference: string): string { - return onTheMachine(reference, held); -} function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } +/** The catalogue's manifest as the lab runs it (harness). This bed's own loader never resolved a + * runtime ARTIFACT to a stocked image, so every module whose runtime the mesh builds travelled to + * the machine unresolved — the shared loader does (novox/hq 04-ISSUES/073). */ function loadManifest(name: string): { manifest: string; broker: boolean } { - const path = resolve(catalogDir, name, "module.json"); - const m = JSON.parse(readFileSync(path, "utf8")) as { - resources?: { type: string; image?: string; ports?: string[] }[]; - }; - const remap = REMAP[name] ?? {}; - for (const r of m.resources ?? []) { - if (r.type !== "container") continue; - if (typeof r.image === "string") r.image = pinned(r.image); - if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); - } - const manifest = JSON.stringify(m); - return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; + const manifest = catalogueModule(name, held, { ports: REMAP[name] }); + return { manifest, broker: needsBrokerAccount(manifest) }; } function tokenFrom(said: string): string { @@ -240,7 +226,6 @@ async function nodeState(node: string): Promise { before(async () => { if (skip) return; - assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), diff --git a/test/integration/whole-mesh-full.test.ts b/test/integration/whole-mesh-full.test.ts index 82f4009..1eb6de1 100644 --- a/test/integration/whole-mesh-full.test.ts +++ b/test/integration/whole-mesh-full.test.ts @@ -58,21 +58,20 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec, instanceNameOf, push } from "../../src/lifecycle/operate.ts"; import { bootstrapBinaryPath, hostBinaryPath, placeBootstrap, BOOTSTRAP_PATH, HOST_PATH, } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, catalogueDir, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import { referenceFor, type HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const installer = bootstrapBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; -const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; // What the installer is told to build. It carries a builder rather than a finished control plane // (novox/hq ADR 0073), so genesis is given a repository and a commit — and a commit rather than a // branch, because what is cloned is the trust anchor for everything this mesh will ever run. @@ -93,7 +92,7 @@ const skip = !capability.usable ? "MESH_LAB_SOURCE is not set to the repository the control plane is built from" : !sourceRef ? "MESH_LAB_SOURCE_REF is not set to the commit to build" - : false; + : catalogueIsPresent(); const SCENARIO = "whole-mesh-full"; /** novox hosts the foundation and the control plane; it is where `mesh` commands run. */ @@ -118,7 +117,7 @@ const CATALOGUE_ON_MACHINE = "/opt/mesh-catalog"; * `internal/bootstrap` RegistryModule and ControlPlaneModule. If it ever opens a third, this list * is where the bed finds out, by the installer saying which manifest it could not read. */ -const CATALOGUE_MODULES = ["registry", "mesh-controller", "builder"]; +const CATALOGUE_MODULES = ["distribution", "mesh-controller", "builder"]; /** * Where this mesh keeps its own images, as the anchor reaches it. @@ -152,9 +151,8 @@ const PUBLIC_DOMAIN: Record = { novox: "novox.incus", ace: "zura const KEEP = !!process.env["MESH_LAB_KEEP"]; const FIXED_ID = process.env["MESH_LAB_INSTANCE_ID"] ?? (KEEP ? "whole-mesh-full-live" : undefined); -const catalogDir = process.env["MESH_LAB_CATALOG"] - ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") - ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); +/** The catalogue's modules directory (harness). Absent, the bed skips — see `skip`. */ +const catalogDir = catalogueIsPresent() ? "" : catalogueDir(); const MEDIA_DIRS = [ "/services/media/series", "/services/media/anime", "/services/media/movies", @@ -201,7 +199,7 @@ const NOVOX: Mod[] = [ // what an operator's `module add` + `assign` would do on a mesh that already has it, and a // module the installer put there had better survive being asked for a second time. It also keeps // mesh-registry in the convergence report, where a reader expects to see it. - { name: "registry", containers: ["mesh-registry"] }, + { name: "distribution", containers: ["mesh-registry"] }, { name: "mailu", containers: [ @@ -210,16 +208,16 @@ const NOVOX: Mod[] = [ "mailu-front", "mesh-mailu", ], }, - { name: "firewall", containers: [], node: true }, + { name: "nftables", containers: [], node: true }, { name: "fail2ban", containers: [], node: true }, ]; const CORE_NOVOX = new Set([ "postgres", "redis", "minio", "mongodb", "mssql", "lavinmq", "route-proxy", "keycloak", "gitea", "nextcloud", "invoicing", "photos", "novox.be", - "portainer", "verdaccio", "registry", + "portainer", "verdaccio", "distribution", ]); const GAPS_NOVOX = new Set([ - "umami", "mailu", "firewall", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", + "umami", "mailu", "nftables", "fail2ban", "only-office", "de-spiegel", "amqp-email-forwarder", // step-ca is reported, not gated: the internal-CA ISSUANCE path is still being fixed in // mesh-controller, and this bed is not the place to discover that a fix has not landed yet. What is // gated is the half that is decided and cheap — see the ADR 0066 section at the end. @@ -348,45 +346,16 @@ async function mesh(command: string, timeoutMs?: number): Promise { } /** What a manifest's image reference becomes on the machine — ours by ID, everything else as written. */ -function pinned(reference: string): string { - return onTheMachine(reference, held); -} function bundleFor(images: HeldImage[]): string { return foundationBundle(bundle, images); } +/** The catalogue's manifest as the lab runs it (harness): artifacts resolved to the images the + * scenario stocked — the lab standing in for the builder — images pinned, host ports remapped. */ function loadManifest(name: string): { manifest: string; broker: boolean } { - const path = resolve(catalogDir, name, "module.json"); - const m = JSON.parse(readFileSync(path, "utf8")) as { - resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[]; - }; - const remap = REMAP[name] ?? {}; - for (const r of m.resources ?? []) { - if (r.type !== "container") continue; - // **A container naming an artifact is a module the mesh builds, and this bed does not build.** - // It pre-builds the same images on the workstation and stocks them, which is the lab standing - // in for the builder — so it does here what the builder does: replace the artifact with the - // reference the machine actually holds. Without this the unresolved field travels to the - // machine, whose declaration language has no such field, and the whole declaration is refused. - // - // The repository is `mesh-runtime-`, which is not a guess: it is what this repository's - // own `scripts/build-module-runtime.sh ` produces and what the scenarios stock by name. - if (typeof r.artifact === "string" && typeof r.image !== "string") { - const reference = referenceFor(held, `mesh-runtime-${name}`); - assert.ok(reference, - `${name} declares the "${r.artifact}" artifact and this scenario stocked no ` + - `mesh-runtime-${name}. The mesh would have to build it, and this bed does not build — ` + - `add it to the machine's images: in the scenario, or build it with ` + - `scripts/build-module-runtime.sh ${name}`); - r.image = reference; - delete r.artifact; - } - if (typeof r.image === "string") r.image = pinned(r.image); - if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); - } - const manifest = JSON.stringify(m); - return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; + const manifest = catalogueModule(name, held, { ports: REMAP[name] }); + return { manifest, broker: needsBrokerAccount(manifest) }; } function tokenFrom(said: string): string { @@ -745,7 +714,6 @@ async function joinTheMesh(): Promise { before(async () => { if (skip) return; - assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), diff --git a/test/integration/whole-mesh-novox.test.ts b/test/integration/whole-mesh-novox.test.ts index daa636a..b44ab47 100644 --- a/test/integration/whole-mesh-novox.test.ts +++ b/test/integration/whole-mesh-novox.test.ts @@ -36,19 +36,17 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { existsSync } from "node:fs"; import { loadScenario } from "../../src/declaration/parse.ts"; import { raise } from "../../src/lifecycle/raise.ts"; import { destroy, exec } from "../../src/lifecycle/operate.ts"; import { hostBinaryPath, HOST_PATH } from "../../src/lifecycle/place.ts"; -import { labIsUsable, destroyAll, foundationBundle, onTheMachine } from "./harness.ts"; +import { labIsUsable, destroyAll, foundationBundle, onTheMachine, catalogueIsPresent, catalogueModule, needsBrokerAccount } from "./harness.ts"; import type { HeldImage } from "../../src/pinning.ts"; const capability = await labIsUsable(); const binary = hostBinaryPath(); const bundle = process.env["MESH_LAB_BUNDLE"] ?? ""; -const modulesEnv = process.env["MESH_LAB_MODULES"] ?? ""; const skip = !capability.usable ? `lab not usable: ${capability.why}` @@ -56,15 +54,12 @@ const skip = !capability.usable ? "MESH_LAB_HOST_BINARY is not set to a built mesh-host" : !bundle || !existsSync(bundle) ? "MESH_LAB_BUNDLE is not set to a foundation bundle (mesh-host examples/)" - : false; + : catalogueIsPresent(); const SCENARIO = "whole-mesh-novox"; const NODE = "novox"; -/** Where the committed module.json files live: the mesh-catalog beside mesh-controller. */ -const catalogDir = process.env["MESH_LAB_CATALOG"] - ?? (modulesEnv ? resolve(dirname(dirname(dirname(modulesEnv))), "mesh-catalog", "modules") : "") - ?? resolve(process.cwd(), "..", "mesh-catalog", "modules"); + /** * The set, in dependency-reading order (the resolver accepts any order). Each row: the module, and @@ -229,38 +224,14 @@ function bundleFor(images: HeldImage[]): string { } /** - * Load a committed module.json, rewrite every container image to the scenario's pinned digest, and - * apply the host-port remaps. Returns the manifest as a string and whether it needs a broker account - * (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules do not). + * The catalogue's manifest as the lab runs it (harness): images pinned, artifacts resolved to the + * stocked images, the host-port remaps applied. Returns the manifest and whether it needs a broker + * account (a runtime that reads MESH_BROKER_FILE — providers and tooled apps do; plain/node modules + * do not). */ function loadManifest(name: string): { manifest: string; broker: boolean } { - const path = resolve(catalogDir, name, "module.json"); - const m = JSON.parse(readFileSync(path, "utf8")) as { - resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[]; - build?: unknown; - }; - const remap = REMAP[name] ?? {}; - for (const r of m.resources ?? []) { - if (r.type !== "container") continue; - if (typeof r.image === "string") { - r.image = pinned(r.image); - } else if (typeof r.artifact === "string") { - // Since issue 060 a module's own runtime container names an artifact the mesh's builder - // would fill, not a placeholder image. This bed stocks the image instead of building, so - // map the artifact to the stocked `mesh-runtime-` the machine holds — keyed on the - // MODULE name, not the container's (mailu's runtime container is `mesh-mailu`, its image is - // `mesh-runtime-mailu`). The placeholder digest is what `pinned` already resolves for a - // mesh-built repo, exactly as it did for the old `image` field. - r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`); - delete r.artifact; - } - if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); - } - // The build section the mesh's builder would consume: dropped, because this bed stocks the image - // rather than building it. Harmless to leave (the push path never reads it), removed for clarity. - delete m.build; - const manifest = JSON.stringify(m); - return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; + const manifest = catalogueModule(name, held, { ports: REMAP[name] }); + return { manifest, broker: needsBrokerAccount(manifest) }; } function tokenFrom(said: string): string { @@ -306,7 +277,6 @@ async function nodeState(node: string): Promise { before(async () => { if (skip) return; - assert.ok(existsSync(catalogDir), `mesh-catalog modules not found at ${catalogDir}`); const raised = await raise(loadScenario(`scenarios/${SCENARIO}.yml`), { onProgress: (m) => console.log(`raise: ${m}`), diff --git a/test/rebuild.test.ts b/test/rebuild.test.ts index 32af5f5..8d7855c 100644 --- a/test/rebuild.test.ts +++ b/test/rebuild.test.ts @@ -104,12 +104,28 @@ test("every repository the receipt claims was built by the run", () => { MESH_LAB_HOST_BINARY: "/repo/host/mesh-host", MESH_LAB_MODULES: "/repo/control/examples/modules", MESH_LAB_BUILDER: "/repo/control/build/mesh-builder", + MESH_LAB_CATALOG: "/repo/catalog/modules", }; const built = new Set(planned(env).map((b) => b.in)); for (const [name, directory] of Object.entries(repositories(env))) { // mesh-lab is the exception, and it is not an omission: it is TypeScript run from source, so // the code under test *is* the code running. There is nothing to build and nothing to go stale. if (name === "mesh-lab") continue; + // mesh-catalog is the other exception, for the other reason: what a bed takes from it is a + // manifest, read from disk when the bed runs. There is nothing built from it that could go + // stale — and claiming it is the whole point, since a bed that carried its own copy of the + // manifest was proving the copy (novox/hq 04-ISSUES/073). + if (name === "mesh-catalog") continue; assert.ok(built.has(directory), `${name} (${directory}) is claimed but never built`); } }); + +// The catalogue is claimed by the receipt, under either spelling the beds accept. +// +// A bed reads the manifest it installs from the catalogue checkout (novox/hq 04-ISSUES/073), so a +// receipt that names no catalogue commit cannot say whether a change there was ever proven. +test("the receipt claims the catalogue the beds read, however it was named", () => { + assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog/modules" })["mesh-catalog"], "/repo/catalog"); + assert.equal(repositories({ MESH_LAB_CATALOG: "/repo/catalog" })["mesh-catalog"], "/repo/catalog"); + assert.equal(repositories({})["mesh-catalog"], undefined); +});