diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 5fcbf54..1bc99d9 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -593,6 +593,12 @@ func one(ctx context.Context, run Runner, publish Publisher, if err != nil { return catalogue.Built{}, fmt.Errorf("%s: writing %s's launchers failed: %w", module, a.Name, err) } + if chain.Bundler != "" { + say("bundle", "bundling each entrypoint into one file") + if compiled, err = bundled(ctx, run, tree, chain, base, a, launchers); err != nil { + return catalogue.Built{}, fmt.Errorf("%s: bundling %s failed: %w", module, a.Name, err) + } + } say("bundle", "compiled, packing") body, err := pack(compiled) if err != nil { @@ -974,7 +980,7 @@ func compile(ctx context.Context, run Runner, tree string, chain Toolchain, if _, err := run(ctx, tree, "docker", invocation...); err != nil { return "", err } - if chain.Dependencies != "" { + if chain.Dependencies != "" && chain.Bundler == "" { // **What the bundle runs with, from the image it was compiled in** (Toolchain.Dependencies). // A second run in the same image rather than a shell wrapped around the compiler: the // compile line stays a plain command a reader can run by hand, and the copy is one more @@ -1224,3 +1230,94 @@ func writeLaunchers(root string, chain Toolchain, a catalogue.Artifact) (map[str } return out, nil } + +// bundledSuffix is where a bundle's one-file output is written, beside what the compiler wrote. +const bundledSuffix = ".bundled" + +// bundled makes every entrypoint and every launcher of a compiled bundle ONE file, in the toolchain +// image's bundler, and answers the directory to pack (novox/hq ADR 0193). +// +// **What a launched bundle runs is what it imports, and nothing else.** Every served bundle is its +// own process, so it carries its own copy of the SDK and its own dependencies inlined — the +// toolchain's whole node_modules no longer travels in every bundle. An entrypoint a process runs by +// name (`node daemon/index.js`) is bundled in place under its own name; a launcher keeps its name +// and its first line, and stays executable. A package the bundler cannot inline is named by the +// artifact (`external`), kept as an import, and only then is the toolchain's runtime directory +// copied beside the files. CommonJS inlined into an ES module still finds `require`. +func bundled(ctx context.Context, run Runner, tree string, chain Toolchain, base string, + a catalogue.Artifact, launchers map[string]string) (string, error) { + const within = "/app/modules/module" + out, final := Out(a.Name), Out(a.Name)+bundledSuffix + if err := os.RemoveAll(filepath.Join(tree, final)); err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Join(tree, final), 0o755); err != nil { + return "", err + } + common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22", + "--outbase=" + out, "--outdir=" + final, "--log-level=warning", + "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"} + for _, x := range a.External { + common = append(common, "--external:"+x) + } + var plain []string + for _, e := range a.Entrypoints { + if strings.HasSuffix(e, ".js") { + plain = append(plain, out+"/"+e) + } + } + var launch []string + for _, l := range sortedValues(launchers) { + launch = append(launch, out+"/"+l) + } + // Refused by name in an image that predates the bundler, as the dependencies copy is: a bundle + // packed without it would carry nothing it imports. + guard := `test -x "$0" || { echo "the toolchain image carries no bundler at $0: it predates one-file bundles, rebuild mesh-tools first" >&2; exit 1; }; exec node "$0" "$@"` + step := func(entries []string, extra ...string) error { + if len(entries) == 0 { + return nil + } + invocation := []string{"run", "--rm", "--volume", tree + ":" + within, "--workdir", within, base, + "sh", "-c", guard, chain.Bundler} + invocation = append(invocation, entries...) + invocation = append(invocation, common...) + invocation = append(invocation, extra...) + _, err := run(ctx, tree, "docker", invocation...) + return err + } + if err := step(plain); err != nil { + return "", err + } + if err := step(launch, "--out-extension:.js=.mjs"); err != nil { + return "", err + } + // Plain `.js` output is an ES module; said once, as the runtime directory used to say it. + if err := os.WriteFile(filepath.Join(tree, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil { + return "", err + } + for _, l := range launchers { + path := filepath.Join(tree, final, filepath.FromSlash(l)) + if _, err := os.Stat(path); err == nil { + if err := os.Chmod(path, 0o755); err != nil { + return "", err + } + } + } + if len(a.External) > 0 && chain.Dependencies != "" { + copying := []string{"run", "--rm", "--volume", tree + ":" + within, "--workdir", within, base, + "sh", "-c", `cp -a "$0/node_modules" "$1/"`, chain.Dependencies, final} + if _, err := run(ctx, tree, "docker", copying...); err != nil { + return "", fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err) + } + } + return filepath.Join(tree, final), nil +} + +func sortedValues(m map[string]string) []string { + out := make([]string, 0, len(m)) + for _, v := range m { + out = append(out, v) + } + sort.Strings(out) + return out +} diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index 7872da2..a29225e 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -83,25 +83,49 @@ func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) } - // **And what it runs with, from the image it was compiled in** (novox/hq to-be 38 WP3). A - // second run in the same toolchain image copies the toolchain's runtime directory — the - // `"type": "module"` package.json and the pruned node_modules — into the output's root, and - // refuses by name when the image carries none rather than packing a bundle that starts nowhere. - var copied string + // **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A + // second run in the same toolchain image bundles each into the artifact's bundled output, the SDK + // inlined, refusing by name in an image that predates the bundler; and the toolchain's + // node_modules is no longer copied into a bundle that keeps nothing external. + var bundling []string for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") && strings.Contains(line, "/app/runtime") { - copied = line + if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") { + bundling = append(bundling, line) } } - if copied == "" { - t.Fatalf("the bundle's dependencies were not copied in after the compile:\n%s", strings.Join(r.ran, "\n")) + if len(bundling) != 2 { + t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n")) } - if !strings.Contains(copied, "mesh-tools/build@sha256:") || !strings.Contains(copied, "predates") || - !strings.Contains(copied, Out("code")) { - t.Fatalf("the copy does not run in the same toolchain, refuse an older image by name, or land in the artifact's output: %s", copied) + for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm", + "--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} { + if !strings.Contains(bundling[0], want) { + t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0]) + } } - if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "/app/runtime") { - t.Fatal("the dependencies were copied before the compile wrote its output") + if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") { + t.Errorf("the launcher is not bundled under its own name: %s", bundling[1]) + } + if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") { + t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n")) + } + if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") { + t.Fatal("the bundler ran before the compile wrote its output") + } +} + +// A bundle naming packages it keeps external is bundled with them as imports, and carries the +// toolchain's node_modules for them — the one case it still does. +func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) { + manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + if _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + all := strings.Join(r.ran, "\n") + if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") { + t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all) } } diff --git a/internal/builder/standing_on_test.go b/internal/builder/standing_on_test.go index 615ce2b..d6462d9 100644 --- a/internal/builder/standing_on_test.go +++ b/internal/builder/standing_on_test.go @@ -200,3 +200,23 @@ func TestWhatABuildReadIsTheRepositoriesItsRecipesName(t *testing.T) { t.Fatal("a module whose recipes name no other repository read one") } } + +// novox/hq 04-ISSUES/212: a toolchain stands on the SDK's published package, and is built with the +// exact version the mesh published — an argument that changes when the SDK does, so a rebuild after +// a release never reuses an install of the version before it. +func TestAPackageTheMeshPublishedIsPassedByItsExactVersion(t *testing.T) { + manifest := catalogue.Manifest{ + Module: "mesh-tools", + Build: &catalogue.Build{ + On: []catalogue.BuildsOn{{Arg: "MESH_SDK", Module: "mesh-sdk", Artifact: "lib"}}, + }, + } + held := map[string]string{"mesh-sdk/lib": "@novox/mesh-sdk@0.1.6"} + args, resolved, err := standingOn(context.Background(), manifest, held, noMirror) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(args) != "[--build-arg MESH_SDK=@novox/mesh-sdk@0.1.6]" || fmt.Sprint(resolved) != "[@novox/mesh-sdk@0.1.6]" { + t.Errorf("the package was passed as %v, recorded as %v", args, resolved) + } +} diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 5df2d3f..72abad6 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -73,7 +73,16 @@ type Toolchain struct { // // A toolchain image without the directory fails the build by name rather than packing a bundle // that starts nowhere: the image predates this and must be rebuilt first. + // + // *Since the bundler (below):* copied only for a bundle that names packages it keeps external, + // which cannot be inlined; a bundle with none carries no node_modules at all. Dependencies string + // Bundler is the bundler inside the toolchain image that makes each compiled entrypoint and each + // launcher ONE self-contained file (novox/hq ADR 0193): every served bundle is its own process + // now, so each carries its own copy of what it imports — the SDK included — and nothing else. + // A bundle shrinks from the toolchain's whole node_modules to the code it runs. Empty for a + // language whose build is already one file. + Bundler string // SystemStamp is the variable this language's linker fills with the artifact's declared system, // for a language whose binaries are pinned to one at link time (novox/hq ADR 0005). // @@ -139,6 +148,7 @@ var toolchains = []Toolchain{ Unit: UnitSources, SourceExt: ".ts", Dependencies: "/app/runtime", + Bundler: "/app/node_modules/esbuild/bin/esbuild", }, { Language: "go", diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 23d3aa6..4db55a1 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -214,6 +214,11 @@ func (b *Build) problems(module string) []string { // A bundle's source is the module's own directory by definition, and what it needs to say // is which compiler — because the mesh chooses that, and cannot choose for a module that // has not said. + if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") { + problems = append(problems, fmt.Sprintf( + "%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+ + "one file with some kept out (novox/hq ADR 0193)", module, a.Name)) + } if len(a.Env) > 0 && a.Kind != ArtifactBundle { problems = append(problems, fmt.Sprintf( "%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+ diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index aef8497..4c55564 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -757,6 +757,11 @@ type Artifact struct { // list twice. A module declaring no tools has nothing the runtime loads, whatever it compiles. Loads []string `json:"loads,omitempty"` + // External are packages a TypeScript bundle keeps as imports rather than inlining — a native + // addon, a package that reads its own files — and so carries the toolchain's node_modules for + // (novox/hq ADR 0193). Absent for nearly every bundle, which is then one file per entrypoint. + External []string `json:"external,omitempty"` + // Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values, // paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment // is, never a secret's content. The node's runtime hands it to this bundle and to no other. diff --git a/internal/inventory/dependencies_test.go b/internal/inventory/dependencies_test.go index 144595a..32c3615 100644 --- a/internal/inventory/dependencies_test.go +++ b/internal/inventory/dependencies_test.go @@ -100,3 +100,23 @@ func TestABundleStandsOnTheToolchainItIsCompiledIn(t *testing.T) { } } } + +// novox/hq 04-ISSUES/212: a toolchain standing on the SDK's package is planned after the SDK, so a +// release of the SDK rebuilds the toolchain, and every bundle compiled in it after that. +func TestAToolchainStandingOnTheSDKFollowsIt(t *testing.T) { + entries := []Entry{ + {Manifest: catalogue.Manifest{Module: "mesh-sdk"}}, + {Manifest: catalogue.Manifest{Module: "mesh-tools", Build: &catalogue.Build{ + On: []catalogue.BuildsOn{{Arg: "MESH_SDK", Module: "mesh-sdk", Artifact: "lib"}}}}}, + } + edges := dependenciesOf(entries, nil, nil) + found := false + for _, e := range edges { + if e.From == "mesh-tools" && e.To == "mesh-sdk" { + found = true + } + } + if !found { + t.Errorf("no edge from the toolchain to the SDK: %v", edges) + } +}