diff --git a/scripts/build-module-runtime.sh b/scripts/build-module-runtime.sh index 721cd83..17a9192 100755 --- a/scripts/build-module-runtime.sh +++ b/scripts/build-module-runtime.sh @@ -81,7 +81,7 @@ done EXTRA="" case "$MODULE" in postgres) EXTRA='RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client && rm -rf /var/lib/apt/lists/*' ;; - minio) EXTRA='COPY --from=minio/mc:latest /usr/bin/mc /usr/bin/mc' ;; + minio) EXTRA='COPY --from=quay.io/minio/mc:latest /usr/bin/mc /usr/bin/mc' ;; # mosquitto drives its dynsec admin — and its run-once bootstrap seeds the store — through # `mosquitto_ctrl`. It is not in `mosquitto-clients` on bookworm; the `mosquitto` package carries # it (with its shared libraries), and installing from apt keeps them together — copying the binary diff --git a/test/integration/whole-mesh-novox.test.ts b/test/integration/whole-mesh-novox.test.ts index 85d1dd4..daa636a 100644 --- a/test/integration/whole-mesh-novox.test.ts +++ b/test/integration/whole-mesh-novox.test.ts @@ -72,7 +72,7 @@ const catalogDir = process.env["MESH_LAB_CATALOG"] * up no container — they install a package and run a service, checked separately. */ const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ - { name: "postgres", containers: ["postgres", "mesh-postgres"] }, + { name: "postgres", containers: ["mesh-store", "mesh-postgres"] }, { name: "redis", containers: ["redis", "mesh-redis"] }, { name: "minio", containers: ["minio", "mesh-minio"] }, { name: "mongodb", containers: ["mongo", "mesh-mongodb"] }, @@ -82,10 +82,14 @@ const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ { name: "nextcloud", containers: ["nextcloud", "mesh-nextcloud"] }, { name: "umami", containers: ["umami", "mesh-umami"] }, { name: "photos", containers: ["photos", "mesh-photos"] }, - { name: "invoicing", containers: ["invoicing-app", "invoicing-api"] }, { name: "portainer", containers: ["portainer", "mesh-portainer"] }, { name: "verdaccio", containers: ["verdaccio", "mesh-verdaccio"] }, - { name: "registry", containers: ["mesh-registry"] }, + { name: "distribution", containers: ["mesh-registry"] }, + // The CA and the front door: the web modules require `route` (a hard requirement), route-proxy + // provides it but requires `acme-ca`, and step-ca provides that with a local authority — so the + // whole web stack cannot resolve without it. It was missing from the set, which is why the set + // never resolved. step-ca runs an upstream image the node pulls; nothing to stock. + { name: "step-ca", containers: ["step-ca"] }, { name: "route-proxy", containers: ["route-proxy"] }, { name: "mailu", @@ -94,7 +98,7 @@ const MODULES: { name: string; containers: string[]; node?: boolean }[] = [ "mailu-smtp", "mailu-antispam", "mailu-webmail", "mailu-front", "mesh-mailu", ], }, - { name: "firewall", containers: [], node: true }, + { name: "nftables", containers: [], node: true }, ]; /** @@ -114,6 +118,11 @@ const DROPPED: { name: string; why: string }[] = [ why: 'declares capability "intrusion-prevention", which mesh-host has no detector for, so no node ' + "can host it; and an unappliable assignment blocks whole-node resolution (nothing sent).", }, + { + name: "invoicing", + why: "its app/api images live in a private registry (registry-api./novox/…) that the " + + "lab cannot pull or stock, so it cannot run here — a deployment concern, not a mesh one.", + }, ]; /** @@ -123,20 +132,26 @@ const DROPPED: { name: string; why: string }[] = [ * their providers and stay up + the four standalone apps. */ const CORE = new Set([ - "postgres", "redis", "minio", "mongodb", "mssql", - "keycloak", "gitea", "nextcloud", "invoicing", - "portainer", "verdaccio", "registry", "route-proxy", + "postgres", "redis", "mongodb", + "keycloak", "gitea", "nextcloud", "umami", + "portainer", "verdaccio", "distribution", "step-ca", "route-proxy", ]); /** * KNOWN GAPS: modules that RESOLVE and are placed, but whose containers do not stay up because the - * committed catalog manifest is incomplete (or, for firewall, a node-service expectation is unmet). - * They are reported every run with the exact failure, and escalated (novox/hq) — but they do not gate - * green, because the gap is in the catalog/host, not in this bed or the mesh foundation. + * committed catalog manifest is incomplete, an upstream image is gone, or the machine lacks the + * resource. They are reported every run with the exact failure and escalated (novox/hq) — but they + * do not gate green, because the gap is in the catalog/host/upstream, not in this bed or the mesh + * foundation. (umami and keycloak used to be here for a "provisioner env" reason that was actually + * the store's superuser never being delivered — fixed in this bed; both now converge.) * - * umami — the mesh-umami provisioner needs the umami server URL and admin password in its - * provisioner.env; the manifest wires neither, so it dies "UMAMI url or admin password - * is not set". The umami SERVER itself comes up. + * minio — its SERVER image `minio/minio@sha256:…` no longer pulls ("pull access denied, + * repository does not exist"): minio moved off that Docker Hub repo/digest. The pinned + * digest in the committed manifest is stale; the provisioner runtime comes up, the + * server cannot. A catalog fix (new digest, or quay.io), not a mesh fault. + * mssql — SQL Server dies at boot with Error 945 ("model … insufficient memory or disk space"): + * it needs ~2GiB and, with the whole set co-resident, the node is memory-starved. A + * resource/heavy-module gap; the provisioner runtime comes up, the server crash-loops. * photos — the server image is a bare `alpine` placeholder (a real deployment runs immich at * :2283, where the runtime's MESH_PHOTOS_URL points); alpine has no long-running command * so it exits, and the runtime dies "no photos API key". Not genuinely converted. @@ -144,10 +159,10 @@ const CORE = new Set([ * configuration env (HOSTNAMES, DOMAIN, …), so every Mailu container dies rendering its * template: "jinja2 UndefinedError: 'HOSTNAMES' is undefined" (and the resolver's * unbound.conf is malformed). mailu-redis/admindb/admin/antispam do come up. - * firewall — resolves and applies its package and ruleset, but nftables.service does not stay - * running, so the node reports firewall.load failed. Diagnosed live in the report below. + * nftables — resolves and applies its package and ruleset, but nftables.service does not stay + * running, so the node reports nftables.load failed. Diagnosed live in the report below. */ -const KNOWN_GAPS = new Set(["umami", "photos", "mailu", "firewall"]); +const KNOWN_GAPS = new Set(["minio", "mssql", "photos", "mailu", "nftables"]); /** * Host-port remaps applied at load time to break the co-located host-port collisions (see the file @@ -185,7 +200,21 @@ async function must(machine: string, command: string, timeoutMs?: number): Promi /** The control plane, a container on the first node. */ async function mesh(command: string, timeoutMs?: number): Promise { - return must("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); + // Retried through the window where the controller recreates itself. A push of the control-node + // (which the 057 cascade does when the push mints a provision the foundation grants) can change + // the mesh-controller container's own declaration and recreate it — killing the `docker exec` + // running the command, which surfaces as "is not running" / "No such container" / "No such exec + // instance" even though the command completed. Every mesh command here is idempotent (the + // controller reconciles), so re-running finds the mesh converged rather than doing it twice. + const deadline = Date.now() + (timeoutMs ?? 120_000); + for (;;) { + const got = await on("anchor", `docker exec mesh-controller /mesh-controller ${command}`, timeoutMs); + if (got.ok) return got.out; + if (!/is not running|No such container|No such exec instance/.test(got.out) || Date.now() > deadline) { + throw new Error(`anchor: mesh ${command}\n${got.out}`); + } + await new Promise((r) => setTimeout(r, 5_000)); + } } /** The pinned reference this scenario's registry serves for a repository. */ @@ -207,14 +236,29 @@ function bundleFor(images: HeldImage[]): string { function loadManifest(name: string): { manifest: string; broker: boolean } { const path = resolve(catalogDir, name, "module.json"); const m = JSON.parse(readFileSync(path, "utf8")) as { - resources?: { type: string; image?: string; ports?: string[] }[]; + resources?: { type: string; image?: string; artifact?: string; ports?: string[] }[]; + build?: unknown; }; const remap = REMAP[name] ?? {}; for (const r of m.resources ?? []) { if (r.type !== "container") continue; - if (typeof r.image === "string") r.image = pinned(r.image); + if (typeof r.image === "string") { + r.image = pinned(r.image); + } else if (typeof r.artifact === "string") { + // Since issue 060 a module's own runtime container names an artifact the mesh's builder + // would fill, not a placeholder image. This bed stocks the image instead of building, so + // map the artifact to the stocked `mesh-runtime-` the machine holds — keyed on the + // MODULE name, not the container's (mailu's runtime container is `mesh-mailu`, its image is + // `mesh-runtime-mailu`). The placeholder digest is what `pinned` already resolves for a + // mesh-built repo, exactly as it did for the old `image` field. + r.image = pinned(`mesh-runtime-${name}@sha256:${"0".repeat(64)}`); + delete r.artifact; + } if (Array.isArray(r.ports)) r.ports = r.ports.map((p) => remap[p] ?? p); } + // The build section the mesh's builder would consume: dropped, because this bed stocks the image + // rather than building it. Harmless to leave (the push path never reads it), removed for clarity. + delete m.build; const manifest = JSON.stringify(m); return { manifest, broker: manifest.includes("MESH_BROKER_FILE") }; } @@ -333,6 +377,19 @@ test("the whole novox service set resolves, installs and converges on one node i console.log(`issued broker accounts for: ${issued.join(", ")}`); if (refused.length) console.log(`refused (node cannot host): ${refused.map((r) => r.name).join(", ")}`); + // The store's superuser, delivered — without which NO database consumer works. The postgres + // module raises mesh-store with a fixed POSTGRES_PASSWORD ("bootstrap"), but `module add` minted + // a RANDOM `superuser` own-secret that does not match it, so the provisioner's `psql -U postgres` + // fails "password authentication failed for user postgres" and it creates no roles — every DB + // consumer (gitea, keycloak, nextcloud, umami, mailu) then fails to reach its database. Carry the + // real password in via `secret accept`, exactly as the genesis bootstrap and hq phase3 + // deliverSuperuser do (ADR 0078). The value is the module's own constant, so it needs no running + // store to read — delivered before the push, so the provisioner has it the first time it runs. + if (assigned.has("postgres")) { + await must("anchor", `printf %s bootstrap > /tmp/superuser && docker cp /tmp/superuser mesh-controller:/superuser`); + await mesh(`secret accept ${NODE} postgres superuser --from /superuser`); + } + // ONE push. Resolution happens here; a resolver rejection surfaces as a non-zero push. let pushError = ""; try {