Merge pull request 'A TypeScript bundle installs its module's own packages before it is compiled (hq ADR 0198 §4)' (#255) from feat/a-bundle-installs-its-own-packages into main

internal/builder/builder.go

@@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher,
 		sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
 		for _, a := range artifacts {
 			say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
-			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say)
+			made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say)
 			if err != nil {
 				say("artifact", "%s FAILED: %v", a.Name, err)
 				return Result{}, err
@@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool {
 
 func one(ctx context.Context, run Runner, publish Publisher,
 	module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string,
-	held map[string]string, npmrc string, seats map[string]string,
+	held map[string]string, npmrc string, registry Npmrc, seats map[string]string,
 	say func(step, format string, args ...any)) (catalogue.Built, error) {
 
 	switch a.Kind {
@@ -582,6 +582,11 @@ func one(ctx context.Context, run Runner, publish Publisher,
 					"holds no copy of it. Build %s first",
 				module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base)
 		}
+		// The module's own packages first, where the compiler and the bundler resolve them from
+		// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
+		if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
+			return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err)
+		}
 		say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base)
 		compiled, err := compile(ctx, run, tree, chain, base, a)
 		if err != nil {

internal/builder/dependencies.go

@@ -0,0 +1,147 @@
+package builder
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// A module's own packages, installed before its bundle is compiled, so the bundler inlines them.
+//
+// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler
+// resolve an import by walking up from the module's source: the module's own directory first, then
+// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a
+// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose
+// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle").
+// Now the module's `package.json` says what it depends on, as any Node package does, and the build
+// installs exactly that into the module's own directory before compiling.
+//
+// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says**
+// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A
+// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a
+// workstation — and installing that range would shadow the toolchain's copy for this module alone:
+// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads.
+// So the SDK is taken out of what is installed (and never fetched), and any copy something else
+// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the
+// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin.
+//
+// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile
+// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept
+// between builds. What is shared is npm's own download cache, a named volume, which is
+// content-addressed and verified by integrity on every read — it saves the network, never the
+// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script
+// would build natively could not be inlined into one file anyway.
+
+// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a
+// module's own dependencies never supply (above).
+const sdkPackage = "@novox/mesh-sdk"
+
+// npmCache is the named volume npm's download cache lives in across builds on one build node.
+const npmCache = "mesh-builder-npm-cache"
+
+// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when
+// the module has no package.json or depends on nothing else — which builds exactly as before.
+func ownDependencies(tree string) ([]string, error) {
+	raw, err := os.ReadFile(filepath.Join(tree, "package.json"))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var p struct {
+		Dependencies map[string]string `json:"dependencies"`
+	}
+	if err := json.Unmarshal(raw, &p); err != nil {
+		return nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
+	}
+	var names []string
+	for name := range p.Dependencies {
+		if name != sdkPackage {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// installSteps is the script run inside the toolchain image, from the module's own directory ($0).
+// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes
+// the SDK out of what is installed, installs production dependencies only, removes any copy of the
+// SDK something pulled in, and puts the result at the module's node_modules.
+const installSteps = `set -e
+work="$(mktemp -d)"
+cp "$0/package.json" "$work/"
+if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi
+cd "$work"
+node -e '
+const fs = require("fs"), sdk = process.argv[1];
+const p = JSON.parse(fs.readFileSync("package.json", "utf8"));
+for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk];
+delete p.devDependencies; delete p.scripts;
+fs.writeFileSync("package.json", JSON.stringify(p));
+' "$1"
+shift
+if [ -f package-lock.json ]; then
+  npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@"
+else
+  npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@"
+fi
+find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} +
+rm -rf "$0/node_modules"
+cp -a node_modules "$0/node_modules"
+`
+
+// installOwn installs a TypeScript module's own production dependencies into its directory, in the
+// toolchain image, before the compile — or does nothing at all for a module that has none.
+func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string,
+	registry Npmrc, say func(step, format string, args ...any)) error {
+	if chain.Language != "typescript" {
+		return nil
+	}
+	deps, err := ownDependencies(tree)
+	if err != nil || len(deps) == 0 {
+		return err
+	}
+	scoped := strings.TrimSpace(registry.Scope)
+	if !registry.Enabled() {
+		// **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on
+		// the public one, where anybody may have published it: a dependency that installs is not
+		// the dependency the module meant.
+		for _, d := range deps {
+			if strings.HasPrefix(d, "@novox/") {
+				return fmt.Errorf("the module depends on %s, and this build knows no package registry "+
+					"for its scope; it would resolve from the public registry, which is not where the "+
+					"mesh publishes it", d)
+			}
+		}
+	}
+	const within = "/app/modules/module"
+	invocation := []string{"run", "--rm",
+		"--volume", tree + ":" + within,
+		"--volume", npmCache + ":/root/.npm",
+		"--workdir", within}
+	var flags []string
+	if registry.Enabled() {
+		// The registry is reached where the binding says it is, which may be this machine's own
+		// loopback — the reason an image build that resolves packages runs on the host network too.
+		invocation = append(invocation, "--network", "host")
+		reg := strings.TrimSpace(registry.Registry)
+		if !strings.HasSuffix(reg, "/") {
+			reg += "/"
+		}
+		flags = append(flags, "--"+scoped+":registry="+reg)
+	}
+	invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage)
+	invocation = append(invocation, flags...)
+	say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", "))
+	if _, err := run(ctx, tree, "docker", invocation...); err != nil {
+		return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err)
+	}
+	return nil
+}

internal/builder/dependencies_test.go

@@ -0,0 +1,131 @@
+package builder
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// A module's own packages (dependencies.go): installed into its own directory, in the toolchain,
+// before the compile, so the bundler inlines them — the SDK always the toolchain's.
+
+func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) {
+	t.Helper()
+	files := map[string]string{"index.ts": "console.log(1)"}
+	if pkg != "" {
+		files["package.json"] = pkg
+	}
+	for k, v := range extra {
+		files[k] = v
+	}
+	r, workspace := aRepository(t, aBundle, files)
+	held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}
+	_, err := Build(context.Background(), compiling{r}.run, r,
+		"https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil)
+	return r, err
+}
+
+func installs(r *recorded) []string {
+	var out []string
+	for _, line := range r.ran {
+		if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") {
+			out = append(out, line)
+		}
+	}
+	return out
+}
+
+func compileIndex(r *recorded) int {
+	for i, line := range r.ran {
+		if strings.Contains(line, "--outDir") {
+			return i
+		}
+	}
+	return -1
+}
+
+func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`,
+		nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := installs(r)
+	if len(got) != 1 {
+		t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n"))
+	}
+	line := got[0]
+	for _, want := range []string{
+		"mesh-tools/build@sha256:",       // in the toolchain image
+		":/app/modules/module",           // into the module's own directory
+		"--workdir /app/modules/module",  //
+		npmCache + ":/root/.npm",         // npm's verified download cache, and only that
+		"--omit=dev", "--ignore-scripts", // production packages, no build-node scripts
+		"npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges
+		"--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry
+		"--network host",
+		"@novox/mesh-sdk", // named, to be taken out of what is installed
+	} {
+		if !strings.Contains(line, want) {
+			t.Errorf("the install lacks %q:\n%s", want, line)
+		}
+	}
+	// The SDK is the toolchain's: never installed from the module's range, and any copy removed.
+	if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) {
+		t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line)
+	}
+	if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 ||
+		i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n"))
+	}
+}
+
+// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install.
+func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) {
+	without, err := buildWithPackageJSON(t, "", nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range []string{
+		`{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`,
+		`{"type":"module"}`,
+	} {
+		with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(strings.Join(with.ran, "\n"), "npm ") {
+			t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n"))
+		}
+		if len(with.ran) != len(without.ran) {
+			t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s",
+				strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n"))
+		}
+	}
+}
+
+// Without the mesh's registry a scoped package would resolve on the public one: refused by name.
+func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) {
+	r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "@novox/other") {
+		t.Fatalf("a scoped package was installed with no registry for its scope: %v", err)
+	}
+	if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") {
+		t.Fatal("the compile ran after the refusal")
+	}
+	// A public package installs without one, from the public registry and nothing else.
+	r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") {
+		t.Fatalf("a public package's install: %v", got)
+	}
+}
+
+func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) {
+	_, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{})
+	if err == nil || !strings.Contains(err.Error(), "package.json") {
+		t.Fatalf("a broken package.json was not refused by name: %v", err)
+	}
+}

internal/builder/toolchain.go

@@ -67,8 +67,9 @@ type Toolchain struct {
 	// `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a
 	// bundle with its dependencies and without that line still fails to start — and the pruned,
 	// production-only node_modules the runtime itself ships with: the SDK's and the runtime's
-	// dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's
+	// dependencies, and nothing module-specific (a module's own npm dependencies are installed into
-	// own npm dependencies are a later step). Empty for a language whose bundle carries its own —
+	// its own directory before the compile and inlined by the bundler: dependencies.go). Empty for a
+	// language whose bundle carries its own —
 	// a Go binary is static, a Python bundle is installed with its dependencies.
 	//
 	// A toolchain image without the directory fails the build by name rather than packing a bundle