From c76c64a4b5ba44effcbf87c008c5ad1dd4343976 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 8 Oct 2026 10:15:59 +0200 Subject: [PATCH] Build a module's runtime image on node-tools, the runtime that replaced mesh-tools' npm package The tool runtime is a Go binary that launches each bundle through the launcher the builder writes; the repository root has no package.json any more, so the old script failed at its first step. --- README.md | 8 +- scripts/build-module-runtime.sh | 172 +++++++++++++++++++++----------- 2 files changed, 121 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 38cbe33..b8c4362 100644 --- a/README.md +++ b/README.md @@ -198,8 +198,12 @@ runs, so the receipt names the catalogue's commit too, and a run taken before a says so (novox/hq 04-ISSUES/073). What IS built from it are the module runtimes the named beds' scenarios stock (`mesh-runtime-:development`): each is compared against the module's source and the tool runtime and SDK it is built on (`MESH_TOOLS`, `MESH_SDK`, or the checkouts -beside this one — they need their `node_modules`), and rebuilt by `scripts/build-module-runtime.sh` -where the image is older, missing, or the source is uncommitted (novox/hq 04-ISSUES/075). +beside this one), and rebuilt by `scripts/build-module-runtime.sh` where the image is older, missing, +or the source is uncommitted (novox/hq 04-ISSUES/075). The image is today's tool runtime: `node-tools`, +built with Go from mesh-tools' `node-tools/`, launching the module's compiled bundle through the +launcher the mesh's builder writes beside each entrypoint (novox/hq ADR 0193) — so building one needs +`go`, and `npm` for the SDK's compiler. It serves on the node's runtime credential or a +`MESH_BROKER_URL`, never the served module's own: the runtime refuses that. `--no-build` skips this too, and then the bed runs whatever image the store holds. Check before running a long suite — it says which of these are missing rather than skipping diff --git a/scripts/build-module-runtime.sh b/scripts/build-module-runtime.sh index 53f57a5..3dbe40a 100755 --- a/scripts/build-module-runtime.sh +++ b/scripts/build-module-runtime.sh @@ -1,10 +1,38 @@ #!/usr/bin/env bash -# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's -# compiled code, which serves that module's tools and runs its events/provisioner under the module's -# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module. +# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built +# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193). # # build-module-runtime.sh # -> tags mesh-runtime-:development and saves it to +# +# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that +# imported every module's compiled code into one process, and this script copied its dist and its +# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/` +# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child +# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes +# beside each compiled entrypoint an executable `.serve.mjs` that imports it and serves what it +# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since, +# so the old script failed at its first step. +# +# This does what the mesh's builder does for a TypeScript bundle, on the workstation: +# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an +# entrypoint lands where it is named); +# 2. writes each entrypoint's launcher, executable; +# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies; +# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the +# module's tools, events and provisioner entrypoints, whichever exist. +# +# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour, +# and is not repeated here. +# +# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it +# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of +# other modules, it is not one). So the container is given the node's runtime credential (the node-tools +# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served +# module's own. +# +# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and +# MESH_CATALOG checkouts. Each defaults to the sibling of this repository. set -euo pipefail MODULE="${1:?usage: build-module-runtime.sh }" @@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)" MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}" MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}" MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}" -MOD="$MESH_CATALOG/modules/$MODULE" +# Absolute, because the steps below run from inside the module. +for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do + [ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; } + printf -v "$v" '%s' "$(cd "${!v}" && pwd)" +done +# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either. +if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi +MOD="$MESH_CATALOG/$MODULE" TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}" BASE="${RUNTIME_BASE:-node:22-bookworm-slim}" +RUNTIME_SRC="$MESH_TOOLS/node-tools" [ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; } +[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; } +command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; } +# The SDK, built: the module compiles against its types and the launchers import its stdio loop. +[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent ) ( cd "$MESH_SDK" && npm run build >/dev/null ) -( cd "$MESH_TOOLS" && npm run build >/dev/null ) -# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools, -# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints -# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage -# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls -# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient -# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the -# ambient declaration is in the program even though the dep is only installed into the image below. -SRCS=(); for f in \ - client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \ - adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \ - pg.d.ts; do - [ -f "$MOD/$f" ] && SRCS+=("$f") + +# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the +# builder reads them; a module declaring none falls back to the ones the runtime serves. +ENTRIES_JS="$(node -e ' + const m = require(process.argv[1]); + const out = new Set(); + for (const a of (m.build && m.build.artifacts) || []) + if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e); + process.stdout.write([...out].join(" ")); +' "$MOD/module.json")" +if [ -z "$ENTRIES_JS" ]; then + for e in tools/index.js index.js provisioner/index.js; do + [ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e" + done +fi +SRCS=(); for e in $ENTRIES_JS; do + [ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; } + SRCS+=("${e%.js}.ts") done +[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; } +# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's +# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image. +for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done + # The module compiles against the SDK, which its package.json names and nothing installs: a module # never built on this workstation has no node_modules, and tsc fails on the first import. Installed -# as a package copy from the sibling checkout (never a link) when absent — the compile needs only -# the types; the image takes the SDK from MESH_SDK below. +# as a package copy from the sibling checkout (never a link) when absent. if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then ( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \ || { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; } fi -# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. -TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 ) STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT -cp -r "$MESH_TOOLS/dist" "$STAGE/dist" -cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules" -# And the sdk, from the sibling this script just built, whatever form the installed tree holds it -# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised — -# true only on a workstation where somebody had linked them, and false the moment the runtime's -# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing -# compiled in it. The image built then looked fine and every entry point inside it pointed at -# nothing. -rm -rf "$STAGE/node_modules/@novox/mesh-sdk" -mkdir -p "$STAGE/node_modules/@novox" -cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk" -rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules" -# **The image runs compiled code and never compiles any**, so it does not need a compiler. The -# tree copied above is the runtime's full install, development dependencies and all — and the -# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing: -# tsc runs on the workstation a few lines above, not in here. -# -# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one -# of them a git URL with no registry behind it — and could drop something the image needs. -rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types" -mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist" -cp "$MESH_TOOLS/package.json" "$STAGE/package.json" +DIST="$STAGE/modules/$MODULE/dist" +# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at +# the module, as the builder compiles, so tools/index.ts lands at tools/index.js. +TSC="$MESH_SDK/node_modules/.bin/tsc" +( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \ + --rootDir . --outDir "$DIST" 1>&2 ) +# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does. +printf '{"type":"module","private":true}\n' > "$DIST/package.json" + +# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder, +# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio. +for e in $ENTRIES_JS; do + [ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; } + launcher="$DIST/${e%.js}.serve.mjs" + cat > "$launcher" </node_modules and still falls back to the shared tree -# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a -# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.) -MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')" +# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from +# /app/modules//node_modules and still falls back to /app/node_modules for the SDK. @novox/* are +# excluded: there is no public registry for them, and the SDK is staged above. +MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")" if [ -n "$MOD_DEPS" ]; then # shellcheck disable=SC2086 npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null fi -# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist. -ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do - [ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e" +# The runtime itself: node-tools, static, from mesh-tools' Go module. +( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools ) + +# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the +# module has — each by its launcher, as =. The others (bootstrap, prepare, apply, …) are +# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for +# every entrypoint. +SERVED=""; for e in tools/index.js index.js provisioner/index.js; do + [ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs" done # A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio @@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" < $OUT" +echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"