Merge pull request 'Compose a module's Go service as a process the host runs (hq issue 213, 1 of 2)' (#252) from fix/issue-213-the-controller-is-a-process into main

A bundle could import only what the toolchain image carried: the compiler and the bundler resolve an import from the module's directory and then the toolchain's node_modules, and nothing ever put anything in the first. So a module needing a database driver (pg, mongodb, mssql) could not be a bundle, and kept a container whose recipe installed it (hq ADR 0198 §4: the backend's own driver inside the bundle).

Now, when a module's package.json depends on anything beyond the SDK, the build installs its production dependencies into the module's directory, in the toolchain image, before the compile: npm ci from the lockfile when there is one, npm install from the ranges otherwise, the mesh's registry for the SDK's scope and the public one for the rest, install scripts off. esbuild then inlines them. A module depending only on the SDK runs exactly the commands it did before.

The SDK stays the toolchain's (hq issue 212): it is taken out of what is installed and any copy something pulls in is removed, so every import of it resolves past the module's node_modules to the one the toolchain carries; a module's own range never shadows it. npm's verified download cache is a named volume; nothing installed is kept between builds. Without a registry, a scoped package is refused rather than resolved on the public registry.

internal/builder/builder.go

@@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
 		sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
 		for _, a := range artifacts {
 			say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
-			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
+			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
 			if err != nil {
 				say("artifact", "%s FAILED: %v", a.Name, err)
 				return Result{}, err
@@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
 
 func one(ctx context.Context, run Runner, publish Publisher,
 	module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
-	held map[string]string, npmrc string, seats map[string]string,
+	held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
 	say func(step, format string, args ...any)) (catalogue.Built, error) {
 
 	switch a.Kind {
@@ -582,6 +582,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
 					"holds no copy of it. Build %s first",
 				module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
 		}
+		// The module's own packages first, where the compiler and the bundler resolve them from
+		// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
+		if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
+			return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+		}
 		say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
 		compiled, err := compile(ctx, run, tree, chain, base, a)
 		if err != nil {

internal/builder/dependencies.go

@@ -0,0 +1,147 @@
+package builder
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// A module's own packages, installed before its bundle is compiled, so the bundler inlines them.
+//
+// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler
+// resolve an import by walking up from the module's source: the module's own directory first, then
+// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a
+// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose
+// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle").
+// Now the module's `package.json` says what it depends on, as any Node package does, and the build
+// installs exactly that into the module's own directory before compiling.
+//
+// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says**
+// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A
+// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a
+// workstation — and installing that range would shadow the toolchain's copy for this module alone:
+// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads.
+// So the SDK is taken out of what is installed (and never fetched), and any copy something else
+// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the
+// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin.
+//
+// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile
+// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept
+// between builds. What is shared is npm's own download cache, a named volume, which is
+// content-addressed and verified by integrity on every read — it saves the network, never the
+// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script
+// would build natively could not be inlined into one file anyway.
+
+// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a
+// module's own dependencies never supply (above).
+const sdkPackage = "@novox/mesh-sdk"
+
+// npmCache is the named volume npm's download cache lives in across builds on one build node.
+const npmCache = "mesh-builder-npm-cache"
+
+// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when
+// the module has no package.json or depends on nothing else — which builds exactly as before.
+func ownDependencies(tree string) ([]string, error) {
+	raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var p struct {
+		Dependencies map[string]string `json:"dependencies"`
+	}
+	if err := json.Unmarshal(raw, &p); err != nil {
+		return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
+	}
+	var names []string
+	for name := range p.Dependencies {
+		if name != sdkPackage {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// installSteps is the script run inside the toolchain image, from the module's own directory ($0).
+// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes
+// the SDK out of what is installed, installs production dependencies only, removes any copy of the
+// SDK something pulled in, and puts the result at the module's node_modules.
+const installSteps = `set -e
+work="$(mktemp -d)"
+cp "$0/package.json" "$work/"
+if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi
+cd "$work"
+node -e '
+const fs = require("fs"), sdk = process.argv[1];
+const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
+for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk];
+delete p.devDependencies; delete p.scripts;
+fs.writeFileSync("package.json", JSON.stringify(p));
+' "$1"
+shift
+if [ -f package-lock.json ]; then
+  npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@"
+else
+  npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@"
+fi
+find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} +
+rm -rf "$0/node_modules"
+cp -a node_modules "$0/node_modules"
+`
+
+// installOwn installs a TypeScript module's own production dependencies into its directory, in the
+// toolchain image, before the compile — or does nothing at all for a module that has none.
+func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
+	registry Npmrc, say func(step, format string, args ...any)) error {
+	if chain.Language != "typescript" {
+		return nil
+	}
+	deps, err := ownDependencies(tree)
+	if err != nil || len(deps) == 0 {
+		return err
+	}
+	scoped := strings.TrimSpace(registry.Scope)
+	if !registry.Enabled() {
+		// **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on
+		// the public one, where anybody may have published it: a dependency that installs is not
+		// the dependency the module meant.
+		for _, d := range deps {
+			if strings.HasPrefix(d, "@novox/") {
+				return fmt.Errorf("the module depends on %s, and this build knows no package registry "+
+					"for its scope; it would resolve from the public registry, which is not where the "+
+					"mesh publishes it", d)
+			}
+		}
+	}
+	const within = "/app/modules/module"
+	invocation := []string{"run", "--rm",
+		"--volume", tree + ":" + within,
+		"--volume", npmCache + ":/root/.npm",
+		"--workdir", within}
+	var flags []string
+	if registry.Enabled() {
+		// The registry is reached where the binding says it is, which may be this machine's own
+		// loopback — the reason an image build that resolves packages runs on the host network too.
+		invocation = append(invocation, "--network", "host")
+		reg := strings.TrimSpace(registry.Registry)
+		if !strings.HasSuffix(reg, "/") {
+			reg += "/"
+		}
+		flags = append(flags, "--"+scoped+":registry="+reg)
+	}
+	invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage)
+	invocation = append(invocation, flags...)
+	say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", "))
+	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
+		return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err)
+	}
+	return nil
+}

internal/builder/dependencies_test.go

@@ -0,0 +1,131 @@
+package builder
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// A module's own packages (dependencies.go): installed into its own directory, in the toolchain,
+// before the compile, so the bundler inlines them — the SDK always the toolchain's.
+
+func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) {
+	t.Helper()
+	files := map[string]string{"index.ts": "console.log(1)"}
+	if pkg != "" {
+		files["package.json"] = pkg
+	}
+	for k, v := range extra {
+		files[k] = v
+	}
+	r, workspace := aRepository(t, aBundle, files)
+	held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
+	_, err := Build(context.Background(), compiling{r}.run, r,
+		"https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil)
+	return r, err
+}
+
+func installs(r *recorded) []string {
+	var out []string
+	for _, line := range r.ran {
+		if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") {
+			out = append(out, line)
+		}
+	}
+	return out
+}
+
+func compileIndex(r *recorded) int {
+	for i, line := range r.ran {
+		if strings.Contains(line, "--outDir") {
+			return i
+		}
+	}
+	return -1
+}
+
+func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`,
+		nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := installs(r)
+	if len(got) != 1 {
+		t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n"))
+	}
+	line := got[0]
+	for _, want := range []string{
+		"mesh-tools/build@sha256:",       // in the toolchain image
+		":/app/modules/module",           // into the module's own directory
+		"--workdir /app/modules/module",  //
+		npmCache + ":/root/.npm",         // npm's verified download cache, and only that
+		"--omit=dev", "--ignore-scripts", // production packages, no build-node scripts
+		"npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges
+		"--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry
+		"--network host",
+		"@novox/mesh-sdk", // named, to be taken out of what is installed
+	} {
+		if !strings.Contains(line, want) {
+			t.Errorf("the install lacks %q:\n%s", want, line)
+		}
+	}
+	// The SDK is the toolchain's: never installed from the module's range, and any copy removed.
+	if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) {
+		t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line)
+	}
+	if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 ||
+		i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n"))
+	}
+}
+
+// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install.
+func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) {
+	without, err := buildWithPackageJSON(t, "", nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range []string{
+		`{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`,
+		`{"type":"module"}`,
+	} {
+		with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(strings.Join(with.ran, "\n"), "npm ") {
+			t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n"))
+		}
+		if len(with.ran) != len(without.ran) {
+			t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s",
+				strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n"))
+		}
+	}
+}
+
+// Without the mesh's registry a scoped package would resolve on the public one: refused by name.
+func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "@novox/other") {
+		t.Fatalf("a scoped package was installed with no registry for its scope: %v", err)
+	}
+	if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatal("the compile ran after the refusal")
+	}
+	// A public package installs without one, from the public registry and nothing else.
+	r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") {
+		t.Fatalf("a public package's install: %v", got)
+	}
+}
+
+func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) {
+	_, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "package.json") {
+		t.Fatalf("a broken package.json was not refused by name: %v", err)
+	}
+}

internal/builder/toolchain.go

@@ -67,8 +67,9 @@ type Toolchain struct {
 	// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
 	// bundle with its dependencies and without that line still fails to start — and the pruned,
 	// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
-	// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
-	// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
+	// dependencies, and nothing module-specific (a module's own npm dependencies are installed into
+	// its own directory before the compile and inlined by the bundler: dependencies.go). Empty for a
+	// language whose bundle carries its own —
 	// a Go binary is static, a Python bundle is installed with its dependencies.
 	//
 	// A toolchain image without the directory fails the build by name rather than packing a bundle