diff --git a/internal/builder/builder.go b/internal/builder/builder.go index ecafc96..5f26afa 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -215,7 +215,7 @@ func Build(ctx context.Context, run Runner, publish Publisher, sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) for _, a := range artifacts { say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a)) - made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, seatBases, say) + made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, say) if err != nil { say("artifact", "%s FAILED: %v", a.Name, err) return Result{}, err @@ -443,7 +443,7 @@ func wantsPackages(manifest catalogue.Manifest, within string) bool { func one(ctx context.Context, run Runner, publish Publisher, module, tree, workspace, commit, credentials string, a catalogue.Artifact, args []string, - held map[string]string, npmrc string, seats map[string]string, + held map[string]string, npmrc string, registry Npmrc, seats map[string]string, say func(step, format string, args ...any)) (catalogue.Built, error) { switch a.Kind { @@ -582,6 +582,11 @@ func one(ctx context.Context, run Runner, publish Publisher, "holds no copy of it. Build %s first", module, a.Name, chain.Language, chain.Base, chain.Artifact, chain.Base) } + // The module's own packages first, where the compiler and the bundler resolve them from + // (dependencies.go); nothing at all for a module whose package.json names only the SDK. + if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil { + return catalogue.Built{}, fmt.Errorf("%s: %s: %w", module, a.Name, err) + } say("bundle", "compiling %s in %s's toolchain", a.Language, chain.Base) compiled, err := compile(ctx, run, tree, chain, base, a) if err != nil { diff --git a/internal/builder/dependencies.go b/internal/builder/dependencies.go new file mode 100644 index 0000000..a11ac56 --- /dev/null +++ b/internal/builder/dependencies.go @@ -0,0 +1,147 @@ +package builder + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// A module's own packages, installed before its bundle is compiled, so the bundler inlines them. +// +// **A bundle could only import what the toolchain happened to carry.** The compiler and the bundler +// resolve an import by walking up from the module's source: the module's own directory first, then +// the toolchain image's node_modules. Nothing ever put anything in the first, so a module needing a +// database driver (`pg`, `mongodb`, `mssql`) could not be a bundle at all, and kept a container whose +// recipe installed it by hand (novox/hq ADR 0198 §4: "the backend's own driver inside the bundle"). +// Now the module's `package.json` says what it depends on, as any Node package does, and the build +// installs exactly that into the module's own directory before compiling. +// +// **The SDK the toolchain carries is the one a bundle is built with, whatever the module says** +// (novox/hq issue 212: the toolchain is rebuilt on every SDK release and every bundle after it). A +// module's `package.json` names `@novox/mesh-sdk` with a range — it has to, to type-check on a +// workstation — and installing that range would shadow the toolchain's copy for this module alone: +// one module compiled against an older SDK than its neighbours, chosen by a caret nobody re-reads. +// So the SDK is taken out of what is installed (and never fetched), and any copy something else +// pulls in is removed afterwards; every import of it resolves past the module's node_modules to the +// toolchain's. A module therefore cannot pin a different SDK, by design: the toolchain is the pin. +// +// **Correctness before speed.** Every build installs afresh into a fresh clone, from the lockfile +// when the module has one (`npm ci`, exact) and from its ranges otherwise; nothing installed is kept +// between builds. What is shared is npm's own download cache, a named volume, which is +// content-addressed and verified by integrity on every read — it saves the network, never the +// install. Install scripts do not run: the build node runs nobody's postinstall, and what a script +// would build natively could not be inlined into one file anyway. + +// sdkPackage is the package a TypeScript bundle's launcher serves through, and the one package a +// module's own dependencies never supply (above). +const sdkPackage = "@novox/mesh-sdk" + +// npmCache is the named volume npm's download cache lives in across builds on one build node. +const npmCache = "mesh-builder-npm-cache" + +// ownDependencies is what a module's package.json depends on beyond the SDK, sorted; nothing when +// the module has no package.json or depends on nothing else — which builds exactly as before. +func ownDependencies(tree string) ([]string, error) { + raw, err := os.ReadFile(filepath.Join(tree, "package.json")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var p struct { + Dependencies map[string]string `json:"dependencies"` + } + if err := json.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("the module's package.json is not JSON: %w", err) + } + var names []string + for name := range p.Dependencies { + if name != sdkPackage { + names = append(names, name) + } + } + sort.Strings(names) + return names, nil +} + +// installSteps is the script run inside the toolchain image, from the module's own directory ($0). +// It works in a scratch copy so the module's package.json and lockfile are never rewritten, takes +// the SDK out of what is installed, installs production dependencies only, removes any copy of the +// SDK something pulled in, and puts the result at the module's node_modules. +const installSteps = `set -e +work="$(mktemp -d)" +cp "$0/package.json" "$work/" +if [ -f "$0/package-lock.json" ]; then cp "$0/package-lock.json" "$work/"; fi +cd "$work" +node -e ' +const fs = require("fs"), sdk = process.argv[1]; +const p = JSON.parse(fs.readFileSync("package.json", "utf8")); +for (const k of ["dependencies", "peerDependencies", "optionalDependencies"]) if (p[k]) delete p[k][sdk]; +delete p.devDependencies; delete p.scripts; +fs.writeFileSync("package.json", JSON.stringify(p)); +' "$1" +shift +if [ -f package-lock.json ]; then + npm ci --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund "$@" +else + npm install --omit=dev --omit=peer --ignore-scripts --no-audit --no-fund --no-package-lock "$@" +fi +find node_modules -depth -type d -path "*/node_modules/@novox/mesh-sdk" -exec rm -rf {} + +rm -rf "$0/node_modules" +cp -a node_modules "$0/node_modules" +` + +// installOwn installs a TypeScript module's own production dependencies into its directory, in the +// toolchain image, before the compile — or does nothing at all for a module that has none. +func installOwn(ctx context.Context, run Runner, tree string, chain Toolchain, base string, + registry Npmrc, say func(step, format string, args ...any)) error { + if chain.Language != "typescript" { + return nil + } + deps, err := ownDependencies(tree) + if err != nil || len(deps) == 0 { + return err + } + scoped := strings.TrimSpace(registry.Scope) + if !registry.Enabled() { + // **No registry, no scoped package.** Without the mesh's registry a scoped name resolves on + // the public one, where anybody may have published it: a dependency that installs is not + // the dependency the module meant. + for _, d := range deps { + if strings.HasPrefix(d, "@novox/") { + return fmt.Errorf("the module depends on %s, and this build knows no package registry "+ + "for its scope; it would resolve from the public registry, which is not where the "+ + "mesh publishes it", d) + } + } + } + const within = "/app/modules/module" + invocation := []string{"run", "--rm", + "--volume", tree + ":" + within, + "--volume", npmCache + ":/root/.npm", + "--workdir", within} + var flags []string + if registry.Enabled() { + // The registry is reached where the binding says it is, which may be this machine's own + // loopback — the reason an image build that resolves packages runs on the host network too. + invocation = append(invocation, "--network", "host") + reg := strings.TrimSpace(registry.Registry) + if !strings.HasSuffix(reg, "/") { + reg += "/" + } + flags = append(flags, "--"+scoped+":registry="+reg) + } + invocation = append(invocation, base, "sh", "-c", installSteps, within, sdkPackage) + invocation = append(invocation, flags...) + say("bundle", "installing the module's own packages: %s", strings.Join(deps, ", ")) + if _, err := run(ctx, tree, "docker", invocation...); err != nil { + return fmt.Errorf("installing the module's own packages (%s): %w", strings.Join(deps, ", "), err) + } + return nil +} diff --git a/internal/builder/dependencies_test.go b/internal/builder/dependencies_test.go new file mode 100644 index 0000000..fbcade2 --- /dev/null +++ b/internal/builder/dependencies_test.go @@ -0,0 +1,131 @@ +package builder + +import ( + "context" + "strings" + "testing" +) + +// A module's own packages (dependencies.go): installed into its own directory, in the toolchain, +// before the compile, so the bundler inlines them — the SDK always the toolchain's. + +func buildWithPackageJSON(t *testing.T, pkg string, extra map[string]string, registry Npmrc) (*recorded, error) { + t.Helper() + files := map[string]string{"index.ts": "console.log(1)"} + if pkg != "" { + files["package.json"] = pkg + } + for k, v := range extra { + files[k] = v + } + r, workspace := aRepository(t, aBundle, files) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, registry, GitCredential{}, nil) + return r, err +} + +func installs(r *recorded) []string { + var out []string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") && strings.Contains(line, "npm ci") { + out = append(out, line) + } + } + return out +} + +func compileIndex(r *recorded) int { + for i, line := range r.ran { + if strings.Contains(line, "--outDir") { + return i + } + } + return -1 +} + +func TestAModulesOwnPackagesAreInstalledInTheToolchainBeforeTheCompile(t *testing.T) { + r, err := buildWithPackageJSON(t, `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0","pg":"^8"},"devDependencies":{"typescript":"^5"}}`, + nil, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm"}) + if err != nil { + t.Fatal(err) + } + got := installs(r) + if len(got) != 1 { + t.Fatalf("want one install of the module's own packages:\n%s", strings.Join(r.ran, "\n")) + } + line := got[0] + for _, want := range []string{ + "mesh-tools/build@sha256:", // in the toolchain image + ":/app/modules/module", // into the module's own directory + "--workdir /app/modules/module", // + npmCache + ":/root/.npm", // npm's verified download cache, and only that + "--omit=dev", "--ignore-scripts", // production packages, no build-node scripts + "npm ci", "npm install", "--no-package-lock", // the lockfile when there is one, else the ranges + "--@novox:registry=https://forge.invalid/api/packages/novox/npm/", // the scope from the mesh's registry + "--network host", + "@novox/mesh-sdk", // named, to be taken out of what is installed + } { + if !strings.Contains(line, want) { + t.Errorf("the install lacks %q:\n%s", want, line) + } + } + // The SDK is the toolchain's: never installed from the module's range, and any copy removed. + if !strings.Contains(line, `delete p[k][sdk]`) || !strings.Contains(line, `-path "*/node_modules/@novox/mesh-sdk" -exec rm -rf`) { + t.Errorf("the module's own SDK range could shadow the toolchain's SDK:\n%s", line) + } + if i, c := strings.Index(strings.Join(r.ran, "\n"), "npm ci"), compileIndex(r); c < 0 || + i > strings.Index(strings.Join(r.ran, "\n"), "--outDir") { + t.Fatalf("the install did not run before the compile:\n%s", strings.Join(r.ran, "\n")) + } +} + +// **A module with nothing beyond the SDK builds exactly as before**: the same commands, no install. +func TestAModuleDependingOnlyOnTheSDKBuildsExactlyAsBefore(t *testing.T) { + without, err := buildWithPackageJSON(t, "", nil, Npmrc{}) + if err != nil { + t.Fatal(err) + } + for _, pkg := range []string{ + `{"type":"module","dependencies":{"@novox/mesh-sdk":"^0.1.0"},"devDependencies":{"typescript":"^5"}}`, + `{"type":"module"}`, + } { + with, err := buildWithPackageJSON(t, pkg, map[string]string{"package-lock.json": "{}"}, Npmrc{Scope: "@novox", Registry: "https://forge.invalid/npm/"}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(strings.Join(with.ran, "\n"), "npm ") { + t.Fatalf("a module depending on nothing but the SDK ran npm:\n%s", strings.Join(with.ran, "\n")) + } + if len(with.ran) != len(without.ran) { + t.Fatalf("a module depending only on the SDK built differently from one with no package.json:\n%s\n---\n%s", + strings.Join(with.ran, "\n"), strings.Join(without.ran, "\n")) + } + } +} + +// Without the mesh's registry a scoped package would resolve on the public one: refused by name. +func TestAScopedPackageWithNoRegistryIsRefused(t *testing.T) { + r, err := buildWithPackageJSON(t, `{"dependencies":{"@novox/mesh-sdk":"^0.1.0","@novox/other":"^1"}}`, nil, Npmrc{}) + if err == nil || !strings.Contains(err.Error(), "@novox/other") { + t.Fatalf("a scoped package was installed with no registry for its scope: %v", err) + } + if strings.Contains(strings.Join(r.ran, "\n"), "--outDir") { + t.Fatal("the compile ran after the refusal") + } + // A public package installs without one, from the public registry and nothing else. + r, err = buildWithPackageJSON(t, `{"dependencies":{"mssql":"^11"}}`, nil, Npmrc{}) + if err != nil { + t.Fatal(err) + } + if got := installs(r); len(got) != 1 || strings.Contains(got[0], ":registry=") || strings.Contains(got[0], "--network host") { + t.Fatalf("a public package's install: %v", got) + } +} + +func TestAnUnreadablePackageJSONIsRefusedByName(t *testing.T) { + _, err := buildWithPackageJSON(t, `{"dependencies":`, nil, Npmrc{}) + if err == nil || !strings.Contains(err.Error(), "package.json") { + t.Fatalf("a broken package.json was not refused by name: %v", err) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 72abad6..db2c4c1 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -67,8 +67,9 @@ type Toolchain struct { // `package.json` saying `"type": "module"` — Node reads a bare `.js` as CommonJS otherwise, so a // bundle with its dependencies and without that line still fails to start — and the pruned, // production-only node_modules the runtime itself ships with: the SDK's and the runtime's - // dependencies, and nothing module-specific yet (novox/hq ADR 0188 §5: a skeleton; a module's - // own npm dependencies are a later step). Empty for a language whose bundle carries its own — + // dependencies, and nothing module-specific (a module's own npm dependencies are installed into + // its own directory before the compile and inlined by the bundler: dependencies.go). Empty for a + // language whose bundle carries its own — // a Go binary is static, a Python bundle is installed with its dependencies. // // A toolchain image without the directory fails the build by name rather than packing a bundle