Build a module's runtime image on node-tools, the runtime that replaced mesh-tools' npm package
The tool runtime is a Go binary that launches each bundle through the launcher the builder writes; the repository root has no package.json any more, so the old script failed at its first step.
This commit is contained in:
jochen
+115
-57
@@ -1,10 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build a per-module runtime image (novox/hq ADR 0052): the tool runtime carrying ONE module's
|
||||
# compiled code, which serves that module's tools and runs its events/provisioner under the module's
|
||||
# own scoped broker account. Generalises build-runtime-image.sh from the audit-logger to any module.
|
||||
# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
|
||||
# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
|
||||
#
|
||||
# build-module-runtime.sh <module> <output.tar>
|
||||
# -> tags mesh-runtime-<module>:development and saves it to <output.tar>
|
||||
#
|
||||
# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
|
||||
# imported every module's compiled code into one process, and this script copied its dist and its
|
||||
# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
|
||||
# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
|
||||
# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
|
||||
# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
|
||||
# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
|
||||
# so the old script failed at its first step.
|
||||
#
|
||||
# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
|
||||
# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
|
||||
# entrypoint lands where it is named);
|
||||
# 2. writes each entrypoint's launcher, executable;
|
||||
# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
|
||||
# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
|
||||
# module's tools, events and provisioner entrypoints, whichever exist.
|
||||
#
|
||||
# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
|
||||
# and is not repeated here.
|
||||
#
|
||||
# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
|
||||
# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
|
||||
# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
|
||||
# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
|
||||
# module's own.
|
||||
#
|
||||
# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
|
||||
# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
|
||||
set -euo pipefail
|
||||
|
||||
MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
|
||||
@@ -13,76 +41,107 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
|
||||
MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
|
||||
MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
|
||||
MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
|
||||
MOD="$MESH_CATALOG/modules/$MODULE"
|
||||
# Absolute, because the steps below run from inside the module.
|
||||
for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
|
||||
[ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
|
||||
printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
|
||||
done
|
||||
# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
|
||||
if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
|
||||
MOD="$MESH_CATALOG/$MODULE"
|
||||
TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
|
||||
BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
|
||||
RUNTIME_SRC="$MESH_TOOLS/node-tools"
|
||||
[ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
|
||||
[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
|
||||
command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
|
||||
|
||||
# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
|
||||
[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
|
||||
( cd "$MESH_SDK" && npm run build >/dev/null )
|
||||
( cd "$MESH_TOOLS" && npm run build >/dev/null )
|
||||
# Compile whichever of the module's entrypoints exist. Besides the serve-time entrypoints (tools,
|
||||
# events, provisioner) and the run-once bootstrap, a module may carry scheduled/one-shot entrypoints
|
||||
# it names in a `schedule`/`run-once` container's args (novox/hq ADR 0052/0053) — refresh/apply/usage
|
||||
# for the anthropic model-access modules, migrate for model-usage's run-once schema step. tsc pulls
|
||||
# in their imports, so leaf files they use are compiled with them. A module may also carry an ambient
|
||||
# `.d.ts` typing a third-party dep it default-imports (model-usage's pg.d.ts) — listed here so the
|
||||
# ambient declaration is in the program even though the dep is only installed into the image below.
|
||||
SRCS=(); for f in \
|
||||
client.ts index.ts tools/index.ts provisioner/index.ts bootstrap/index.ts \
|
||||
adopt/index.ts refresh/index.ts apply/index.ts usage/index.ts migrate/index.ts \
|
||||
pg.d.ts; do
|
||||
[ -f "$MOD/$f" ] && SRCS+=("$f")
|
||||
|
||||
# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
|
||||
# builder reads them; a module declaring none falls back to the ones the runtime serves.
|
||||
ENTRIES_JS="$(node -e '
|
||||
const m = require(process.argv[1]);
|
||||
const out = new Set();
|
||||
for (const a of (m.build && m.build.artifacts) || [])
|
||||
if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
|
||||
process.stdout.write([...out].join(" "));
|
||||
' "$MOD/module.json")"
|
||||
if [ -z "$ENTRIES_JS" ]; then
|
||||
for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
|
||||
done
|
||||
fi
|
||||
SRCS=(); for e in $ENTRIES_JS; do
|
||||
[ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
|
||||
SRCS+=("${e%.js}.ts")
|
||||
done
|
||||
[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
|
||||
# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
|
||||
# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
|
||||
for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
|
||||
|
||||
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
||||
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
||||
# as a package copy from the sibling checkout (never a link) when absent — the compile needs only
|
||||
# the types; the image takes the SDK from MESH_SDK below.
|
||||
# as a package copy from the sibling checkout (never a link) when absent.
|
||||
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
||||
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
||||
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
||||
fi
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"; ( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist 1>&2 )
|
||||
|
||||
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
||||
cp -r "$MESH_TOOLS/dist" "$STAGE/dist"
|
||||
cp -rL "$MESH_TOOLS/node_modules" "$STAGE/node_modules"
|
||||
# And the sdk, from the sibling this script just built, whatever form the installed tree holds it
|
||||
# in. It used to be relied on being a symlink into that sibling, which `-L` above materialised —
|
||||
# true only on a workstation where somebody had linked them, and false the moment the runtime's
|
||||
# dependencies are installed the ordinary way, which now fetches the sdk as sources with nothing
|
||||
# compiled in it. The image built then looked fine and every entry point inside it pointed at
|
||||
# nothing.
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
mkdir -p "$STAGE/node_modules/@novox"
|
||||
cp -rL "$MESH_SDK" "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
rm -rf "$STAGE/node_modules/@novox/mesh-sdk/node_modules"
|
||||
# **The image runs compiled code and never compiles any**, so it does not need a compiler. The
|
||||
# tree copied above is the runtime's full install, development dependencies and all — and the
|
||||
# compiler alone is 23 of its 28 MB. Every module image carried one, on every machine, for nothing:
|
||||
# tsc runs on the workstation a few lines above, not in here.
|
||||
#
|
||||
# Removed by name rather than by `npm prune --omit=dev`, which would re-resolve dependencies — one
|
||||
# of them a git URL with no registry behind it — and could drop something the image needs.
|
||||
rm -rf "$STAGE/node_modules/typescript" "$STAGE/node_modules/@types"
|
||||
mkdir -p "$STAGE/modules/$MODULE"; cp -r "$MOD/dist" "$STAGE/modules/$MODULE/dist"
|
||||
cp "$MESH_TOOLS/package.json" "$STAGE/package.json"
|
||||
DIST="$STAGE/modules/$MODULE/dist"
|
||||
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
|
||||
# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
|
||||
TSC="$MESH_SDK/node_modules/.bin/tsc"
|
||||
( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
|
||||
--rootDir . --outDir "$DIST" 1>&2 )
|
||||
# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
|
||||
printf '{"type":"module","private":true}\n' > "$DIST/package.json"
|
||||
|
||||
# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
|
||||
# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
|
||||
for e in $ENTRIES_JS; do
|
||||
[ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
|
||||
launcher="$DIST/${e%.js}.serve.mjs"
|
||||
cat > "$launcher" <<LAUNCHER
|
||||
#!/usr/bin/env node
|
||||
// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
|
||||
// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
|
||||
import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
|
||||
await import("./$(basename "$e")");
|
||||
await serveRegisteredOverStdio();
|
||||
LAUNCHER
|
||||
chmod 0755 "$launcher"
|
||||
done
|
||||
|
||||
# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
|
||||
# runtime dependencies of its own.
|
||||
mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
|
||||
cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
|
||||
cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
|
||||
|
||||
# A module may declare its own third-party runtime deps (the anthropic-manager seals with
|
||||
# tweetnacl-sealedbox-js). The shared node_modules copied above carries the common packages and
|
||||
# @novox/* — but not a module's private deps. Install those under the module itself, so Node
|
||||
# resolves them from /app/modules/<module>/node_modules and still falls back to the shared tree
|
||||
# at /app/node_modules for @novox/* and everything common. Modules with no non-@novox deps are a
|
||||
# no-op. (@novox/* are workspace deps with no registry to fetch from, so they are excluded here.)
|
||||
MOD_DEPS="$(node -e 'const d=(require("'"$MOD"'/package.json").dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))')"
|
||||
# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
|
||||
# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
|
||||
# excluded: there is no public registry for them, and the SDK is staged above.
|
||||
MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
|
||||
if [ -n "$MOD_DEPS" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
|
||||
fi
|
||||
|
||||
# The entrypoints the runtime loads: tools, events and (a provider's) provisioner, whichever exist.
|
||||
ENTRIES=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$STAGE/modules/$MODULE/dist/$e" ] && ENTRIES="${ENTRIES:+$ENTRIES,}/app/modules/$MODULE/dist/$e"
|
||||
# The runtime itself: node-tools, static, from mesh-tools' Go module.
|
||||
( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
|
||||
|
||||
# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
|
||||
# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
|
||||
# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
|
||||
# every entrypoint.
|
||||
SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
|
||||
[ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
|
||||
done
|
||||
|
||||
# A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
|
||||
@@ -105,13 +164,12 @@ cat > "$STAGE/Dockerfile" <<DOCKER
|
||||
FROM $BASE
|
||||
WORKDIR /app
|
||||
$EXTRA
|
||||
COPY package.json ./
|
||||
COPY node-tools /usr/local/bin/node-tools
|
||||
COPY node_modules ./node_modules
|
||||
COPY dist ./dist
|
||||
COPY modules ./modules
|
||||
ENV MESH_TOOL_MODULES=$ENTRIES
|
||||
ENTRYPOINT ["node", "dist/main.js"]
|
||||
ENV MESH_TOOL_MODULES=$SERVED
|
||||
ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
|
||||
DOCKER
|
||||
docker build -t "$TAG" "$STAGE"
|
||||
docker save -o "$OUT" "$TAG"
|
||||
echo "built $TAG (entrypoints: $ENTRIES) -> $OUT"
|
||||
echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"
|
||||
|
||||
Reference in New Issue
Block a user