jochen
c76c64a4b5
The tool runtime is a Go binary that launches each bundle through the launcher the builder writes; the repository root has no package.json any more, so the old script failed at its first step.
176 lines
10 KiB
Bash
Executable File
176 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Build a per-module runtime image: the node's tool runtime serving ONE module's tools bundle, built
|
|
# the way the mesh builds and serves it today (novox/hq ADR 0175, 0188, 0193).
|
|
#
|
|
# build-module-runtime.sh <module> <output.tar>
|
|
# -> tags mesh-runtime-<module>:development and saves it to <output.tar>
|
|
#
|
|
# **What replaced mesh-tools' npm package.** The tool runtime was a TypeScript program in mesh-tools that
|
|
# imported every module's compiled code into one process, and this script copied its dist and its
|
|
# node_modules into an image. The runtime is now `node-tools` — a Go binary in mesh-tools' `node-tools/`
|
|
# (built by the mesh as that module's bundle) — which imports nothing: it LAUNCHES each bundle as a child
|
|
# and speaks MCP over stdio to it. A TypeScript bundle is made launchable by the builder, which writes
|
|
# beside each compiled entrypoint an executable `<entry>.serve.mjs` that imports it and serves what it
|
|
# registered through the SDK's `@novox/mesh-sdk/stdio`. The repository root has had no package.json since,
|
|
# so the old script failed at its first step.
|
|
#
|
|
# This does what the mesh's builder does for a TypeScript bundle, on the workstation:
|
|
# 1. compiles the module's declared entrypoints against the sibling SDK (rooted at the module, so an
|
|
# entrypoint lands where it is named);
|
|
# 2. writes each entrypoint's launcher, executable;
|
|
# 3. stages the SDK (it has no runtime dependencies) and the module's own third-party dependencies;
|
|
# 4. builds node-tools from mesh-tools' Go module, and makes it the image's entrypoint, serving the
|
|
# module's tools, events and provisioner entrypoints, whichever exist.
|
|
#
|
|
# The builder also bundles each file into one with esbuild; that is a size optimisation, not a behaviour,
|
|
# and is not repeated here.
|
|
#
|
|
# **Which credential the image runs on.** node-tools serves the modules it is GIVEN, on the credential it
|
|
# holds, and refuses a module that is the credential's own (ADR 0193: the runtime launches bundles of
|
|
# other modules, it is not one). So the container is given the node's runtime credential (the node-tools
|
|
# module's, as the mesh issues it) in MESH_BROKER_FILE, or a plain MESH_BROKER_URL — never the served
|
|
# module's own.
|
|
#
|
|
# Needs: go, node and npm; MESH_SDK with its node_modules (for the compiler); MESH_TOOLS and
|
|
# MESH_CATALOG checkouts. Each defaults to the sibling of this repository.
|
|
set -euo pipefail
|
|
|
|
MODULE="${1:?usage: build-module-runtime.sh <module> <output.tar>}"
|
|
OUT="${2:?usage: build-module-runtime.sh <module> <output.tar>}"
|
|
HERE="$(cd "$(dirname "$0")/.." && pwd)"; ROOT="$(cd "$HERE/.." && pwd)"
|
|
MESH_TOOLS="${MESH_TOOLS:-$ROOT/mesh-tools}"
|
|
MESH_SDK="${MESH_SDK:-$ROOT/mesh-sdk}"
|
|
MESH_CATALOG="${MESH_CATALOG:-$ROOT/mesh-catalog}"
|
|
# Absolute, because the steps below run from inside the module.
|
|
for v in MESH_TOOLS MESH_SDK MESH_CATALOG; do
|
|
[ -d "${!v}" ] || { echo "$v=${!v} is not a checkout" >&2; exit 1; }
|
|
printf -v "$v" '%s' "$(cd "${!v}" && pwd)"
|
|
done
|
|
# A catalogue checkout or its modules/ directory, as MESH_LAB_CATALOG may name either.
|
|
if [ -d "$MESH_CATALOG/modules" ]; then MESH_CATALOG="$MESH_CATALOG/modules"; fi
|
|
MOD="$MESH_CATALOG/$MODULE"
|
|
TAG="${RUNTIME_TAG:-mesh-runtime-$MODULE:development}"
|
|
BASE="${RUNTIME_BASE:-node:22-bookworm-slim}"
|
|
RUNTIME_SRC="$MESH_TOOLS/node-tools"
|
|
[ -d "$MOD" ] || { echo "no module $MODULE at $MOD" >&2; exit 1; }
|
|
[ -f "$RUNTIME_SRC/go.mod" ] || { echo "no node-tools Go module at $RUNTIME_SRC (MESH_TOOLS=$MESH_TOOLS)" >&2; exit 1; }
|
|
command -v go >/dev/null || { echo "go is needed to build node-tools, the runtime the image runs" >&2; exit 1; }
|
|
|
|
# The SDK, built: the module compiles against its types and the launchers import its stdio loop.
|
|
[ -d "$MESH_SDK/node_modules" ] || ( cd "$MESH_SDK" && npm ci --no-audit --no-fund --silent )
|
|
( cd "$MESH_SDK" && npm run build >/dev/null )
|
|
|
|
# The entrypoints the module declares for its TypeScript bundle (build.artifacts[].entrypoints), as the
|
|
# builder reads them; a module declaring none falls back to the ones the runtime serves.
|
|
ENTRIES_JS="$(node -e '
|
|
const m = require(process.argv[1]);
|
|
const out = new Set();
|
|
for (const a of (m.build && m.build.artifacts) || [])
|
|
if (a.language === "typescript") for (const e of a.entrypoints || []) if (e.endsWith(".js")) out.add(e);
|
|
process.stdout.write([...out].join(" "));
|
|
' "$MOD/module.json")"
|
|
if [ -z "$ENTRIES_JS" ]; then
|
|
for e in tools/index.js index.js provisioner/index.js; do
|
|
[ -f "$MOD/${e%.js}.ts" ] && ENTRIES_JS="$ENTRIES_JS $e"
|
|
done
|
|
fi
|
|
SRCS=(); for e in $ENTRIES_JS; do
|
|
[ -f "$MOD/${e%.js}.ts" ] || { echo "$MODULE declares $e and has no ${e%.js}.ts" >&2; exit 1; }
|
|
SRCS+=("${e%.js}.ts")
|
|
done
|
|
[ "${#SRCS[@]}" -gt 0 ] || { echo "$MODULE has no TypeScript entrypoint to serve" >&2; exit 1; }
|
|
# An ambient `.d.ts` at the module's root types a third-party dep it default-imports (model-usage's
|
|
# pg.d.ts); in the program so the compile sees it, though the dep is installed only into the image.
|
|
for d in "$MOD"/*.d.ts; do [ -f "$d" ] && SRCS+=("$(basename "$d")"); done
|
|
|
|
# The module compiles against the SDK, which its package.json names and nothing installs: a module
|
|
# never built on this workstation has no node_modules, and tsc fails on the first import. Installed
|
|
# as a package copy from the sibling checkout (never a link) when absent.
|
|
if [ ! -e "$MOD/node_modules/@novox/mesh-sdk" ]; then
|
|
( cd "$MOD" && npm install --no-save --install-links --no-package-lock --ignore-scripts --silent "$MESH_SDK" ) \
|
|
|| { echo "cannot install the SDK into $MOD for the compile" >&2; exit 1; }
|
|
fi
|
|
|
|
STAGE="$(mktemp -d)"; trap 'rm -rf "$STAGE"' EXIT
|
|
DIST="$STAGE/modules/$MODULE/dist"
|
|
# Output kept: a compile error hidden behind /dev/null is a build that fails saying nothing. Rooted at
|
|
# the module, as the builder compiles, so tools/index.ts lands at tools/index.js.
|
|
TSC="$MESH_SDK/node_modules/.bin/tsc"
|
|
( cd "$MOD" && "$TSC" "${SRCS[@]}" --module NodeNext --moduleResolution NodeNext --target ES2022 \
|
|
--rootDir . --outDir "$DIST" 1>&2 )
|
|
# The compiled files are ES modules; said where Node looks for it, as the builder's bundle does.
|
|
printf '{"type":"module","private":true}\n' > "$DIST/package.json"
|
|
|
|
# A launcher beside every entrypoint, exactly the builder's (mesh-controller internal/builder,
|
|
# writeLaunchers): node-tools starts it and it serves what the entrypoint registered over stdio.
|
|
for e in $ENTRIES_JS; do
|
|
[ -f "$DIST/$e" ] || { echo "the compile wrote no $e" >&2; exit 1; }
|
|
launcher="$DIST/${e%.js}.serve.mjs"
|
|
cat > "$launcher" <<LAUNCHER
|
|
#!/usr/bin/env node
|
|
// Written as the mesh's builder writes it (novox/hq ADR 0193): serve what $e registers,
|
|
// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.
|
|
import { serveRegisteredOverStdio } from "@novox/mesh-sdk/stdio";
|
|
await import("./$(basename "$e")");
|
|
await serveRegisteredOverStdio();
|
|
LAUNCHER
|
|
chmod 0755 "$launcher"
|
|
done
|
|
|
|
# The SDK, from the sibling just built, as a package copy: its package.json and its dist. It has no
|
|
# runtime dependencies of its own.
|
|
mkdir -p "$STAGE/node_modules/@novox/mesh-sdk"
|
|
cp "$MESH_SDK/package.json" "$STAGE/node_modules/@novox/mesh-sdk/"
|
|
cp -r "$MESH_SDK/dist" "$STAGE/node_modules/@novox/mesh-sdk/dist"
|
|
|
|
# A module may declare its own third-party runtime deps (the anthropic-manager seals with
|
|
# tweetnacl-sealedbox-js). Installed under the module, so Node resolves them from
|
|
# /app/modules/<module>/node_modules and still falls back to /app/node_modules for the SDK. @novox/* are
|
|
# excluded: there is no public registry for them, and the SDK is staged above.
|
|
MOD_DEPS="$(node -e 'const d=(require(process.argv[1]).dependencies)||{};process.stdout.write(Object.keys(d).filter(k=>!k.startsWith("@novox/")).map(k=>k+"@"+d[k]).join(" "))' "$MOD/package.json")"
|
|
if [ -n "$MOD_DEPS" ]; then
|
|
# shellcheck disable=SC2086
|
|
npm install --prefix "$STAGE/modules/$MODULE" --omit=dev --no-save --no-package-lock --ignore-scripts $MOD_DEPS >/dev/null
|
|
fi
|
|
|
|
# The runtime itself: node-tools, static, from mesh-tools' Go module.
|
|
( cd "$RUNTIME_SRC" && CGO_ENABLED=0 go build -trimpath -o "$STAGE/node-tools" ./cmd/node-tools )
|
|
|
|
# What the runtime serves: the tools, events and (a provider's) provisioner entrypoints, whichever the
|
|
# module has — each by its launcher, as <module>=<path>. The others (bootstrap, prepare, apply, …) are
|
|
# run once by name, as the mesh runs them, and carry launchers only because the builder writes one for
|
|
# every entrypoint.
|
|
SERVED=""; for e in tools/index.js index.js provisioner/index.js; do
|
|
[ -f "$DIST/${e%.js}.serve.mjs" ] && SERVED="${SERVED:+$SERVED,}$MODULE=/app/modules/$MODULE/dist/${e%.js}.serve.mjs"
|
|
done
|
|
|
|
# A module whose code drives a CLI needs that CLI in the image — postgres shells out to `psql`, minio
|
|
# to `mc`. Everything else speaks a wire protocol or HTTP and needs nothing added.
|
|
EXTRA=""
|
|
case "$MODULE" in
|
|
postgres) EXTRA='RUN apt-get update && apt-get install -y --no-install-recommends postgresql-client && rm -rf /var/lib/apt/lists/*' ;;
|
|
minio) EXTRA='COPY --from=quay.io/minio/mc:latest /usr/bin/mc /usr/bin/mc' ;;
|
|
# mosquitto drives its dynsec admin — and its run-once bootstrap seeds the store — through
|
|
# `mosquitto_ctrl`. It is not in `mosquitto-clients` on bookworm; the `mosquitto` package carries
|
|
# it (with its shared libraries), and installing from apt keeps them together — copying the binary
|
|
# out of the (musl) eclipse-mosquitto image into this (glibc) base would not load.
|
|
mosquitto) EXTRA='RUN apt-get update && apt-get install -y --no-install-recommends mosquitto && rm -rf /var/lib/apt/lists/*' ;;
|
|
# mongodb's client shells out to `mongosh`. Install it from MongoDB's own apt repo so its shared
|
|
# libraries come with it — copying just the binary out of the mongo image leaves it unable to load.
|
|
mongodb) EXTRA='RUN apt-get update && apt-get install -y --no-install-recommends gnupg curl ca-certificates && curl -fsSL https://pgp.mongodb.com/server-7.0.asc | gpg --dearmor -o /usr/share/keyrings/mongodb.gpg && echo "deb [signed-by=/usr/share/keyrings/mongodb.gpg] https://repo.mongodb.org/apt/debian bookworm/mongodb-org/7.0 main" > /etc/apt/sources.list.d/mongodb.list && apt-get update && apt-get install -y --no-install-recommends mongodb-mongosh && rm -rf /var/lib/apt/lists/*' ;;
|
|
esac
|
|
|
|
cat > "$STAGE/Dockerfile" <<DOCKER
|
|
FROM $BASE
|
|
WORKDIR /app
|
|
$EXTRA
|
|
COPY node-tools /usr/local/bin/node-tools
|
|
COPY node_modules ./node_modules
|
|
COPY modules ./modules
|
|
ENV MESH_TOOL_MODULES=$SERVED
|
|
ENTRYPOINT ["/usr/local/bin/node-tools", "serve"]
|
|
DOCKER
|
|
docker build -t "$TAG" "$STAGE"
|
|
docker save -o "$OUT" "$TAG"
|
|
echo "built $TAG (serving: ${SERVED:-nothing}) -> $OUT"
|