Add the Node build module's manifest, ssh files, toolchain package, build step and toolchain tests (issues 1488, 1484, 1490), part 15 of 19
This commit is contained in:
@@ -0,0 +1,266 @@
|
||||
package main
|
||||
|
||||
// The build step (novox/hq ADR 0405 §3 and §7, ADR 0469 §4; issue 1490): a TypeScript bundle of a fetched workspace
|
||||
// compiled with the toolchain's tsc, bundled with its esbuild, given its launchers, packed with mesh-sdk's Pack and
|
||||
// ended with the archive's digest. No network. The commands and flags are the build agent's, copied from
|
||||
// mesh-controller origin/main internal/builder: toolchain.go:143-147 (tsc), builder.go:1637-1705 (compile),
|
||||
// :1926-1956 (writeLaunchers), :1971-2039 (bundled), :1735-1747 (sourcesFor). Where they ran in the toolchain image
|
||||
// they run here from the module's directory with the toolchain's real paths, and tsc and esbuild find the SDK in the
|
||||
// workspace's node_modules (workspace.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// launcherSuffix is what an entrypoint's launcher is called beside it: index.js is started as index.serve.mjs
|
||||
// (novox/hq ADR 0193).
|
||||
const launcherSuffix = ".serve.mjs"
|
||||
|
||||
// Out is where a bundle's compiled files go in the module's directory (builder toolchain.go:118).
|
||||
func Out(artifact string) string { return ".mesh-build/" + artifact }
|
||||
|
||||
// banner is esbuild's createRequire shim: CommonJS inlined into an ES module still finds `require`.
|
||||
const banner = "--banner:js=import { createRequire as __meshRequire } from 'node:module'; const require = __meshRequire(import.meta.url);"
|
||||
|
||||
// artifact is the part of a module.json artifact the build reads.
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Language string `json:"language"`
|
||||
Entrypoints []string `json:"entrypoints"`
|
||||
External []string `json:"external"`
|
||||
}
|
||||
|
||||
// BuildResult is what a passed build step says.
|
||||
type BuildResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Module string `json:"module"`
|
||||
Artifact string `json:"artifact"`
|
||||
// Digest is "sha256:" and the hex of the archive's bytes (buildstep.Pack).
|
||||
Digest string `json:"digest"`
|
||||
Bytes int `json:"bytes"`
|
||||
// Archive is the packed file, in the workspace beside its tree.
|
||||
Archive string `json:"archive"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_build",
|
||||
Description: "Build a TypeScript bundle of a fetched workspace: compile it with the toolchain's tsc, bundle each " +
|
||||
"entrypoint with its esbuild, write the launchers and package.json, pack the bundle and answer its digest. " +
|
||||
"Answers a step id at once; the step ends on its key in the steps state with the digest. No network.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, '.' when it is the root"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the name of the bundle in the module's module.json"},
|
||||
},
|
||||
"required": []string{"workspace", "artifact"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.build,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
root, dir, err := m.moduleDirOf(workspace, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := readArtifact(dir, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.provide(root); err != nil {
|
||||
return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err)
|
||||
}
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.MkdirAll(filepath.Join(dir, out), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// tsc: builder.go:1637-1705 with toolchain.go:143-147's flags, rooted at the module so an entrypoint lands where
|
||||
// it is named.
|
||||
tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022",
|
||||
"--rootDir", ".", "--outDir", out}
|
||||
tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, "node", tsc...)); why != "" {
|
||||
return nil, fmt.Errorf("tsc for %s: %s", a.Name, why)
|
||||
}
|
||||
launchers, err := writeLaunchers(filepath.Join(dir, out), a)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.bundle(ctx, dir, a, launchers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bundleDir := filepath.Join(dir, final)
|
||||
if len(a.External) > 0 {
|
||||
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(bundleDir, "node_modules")); err != nil {
|
||||
return nil, fmt.Errorf("copying the packages %s keeps external: %w", a.Name, err)
|
||||
}
|
||||
}
|
||||
archive, digest, err := buildstep.Pack(bundleDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("packing %s: %w", a.Name, err)
|
||||
}
|
||||
file := filepath.Join(root, a.Name+".tar.gz")
|
||||
if err := os.WriteFile(file, archive, 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := []string{}
|
||||
_ = filepath.WalkDir(bundleDir, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
rel, _ := filepath.Rel(bundleDir, p)
|
||||
files = append(files, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
return BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: digest, Bytes: len(archive), Archive: file, Files: files}, nil
|
||||
}
|
||||
|
||||
// bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is
|
||||
// bundled in place under its own name, a launcher keeps its name with the extension .mjs.
|
||||
func (m *Module) bundle(ctx context.Context, dir string, a artifact, launchers map[string]string) error {
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.RemoveAll(filepath.Join(dir, final)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, final), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
common := []string{"--bundle", "--platform=node", "--format=esm", "--target=node22",
|
||||
"--outbase=" + out, "--outdir=" + final, "--log-level=warning", banner}
|
||||
for _, x := range a.External {
|
||||
common = append(common, "--external:"+x)
|
||||
}
|
||||
var plain, launch []string
|
||||
for _, e := range a.Entrypoints {
|
||||
if strings.HasSuffix(e, ".js") {
|
||||
plain = append(plain, out+"/"+e)
|
||||
}
|
||||
}
|
||||
for _, l := range sortedValues(launchers) {
|
||||
launch = append(launch, out+"/"+l)
|
||||
}
|
||||
step := func(entries []string, extra ...string) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := append(append(append([]string{}, entries...), common...), extra...)
|
||||
if why := ranFailed(m.runIn(ctx, dir, nil, m.esbuild(), args...)); why != "" {
|
||||
return fmt.Errorf("esbuild for %s: %s", a.Name, why)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := step(plain); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := step(launch, "--out-extension:.js=.mjs"); err != nil {
|
||||
return err
|
||||
}
|
||||
// Plain `.js` output is an ES module; said once.
|
||||
if err := os.WriteFile(filepath.Join(dir, final, "package.json"), []byte(`{"type":"module","private":true}`+"\n"), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, l := range launchers {
|
||||
path := filepath.Join(dir, final, filepath.FromSlash(l))
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeLaunchers writes, beside every entrypoint, an executable that imports the entrypoint and serves what it
|
||||
// registered over MCP on stdio (builder.go:1926-1956; novox/hq ADR 0193). Its answer is each entrypoint's launcher,
|
||||
// relative to the bundle's root.
|
||||
func writeLaunchers(root string, a artifact) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, entry := range a.Entrypoints {
|
||||
if !strings.HasSuffix(entry, ".js") {
|
||||
continue
|
||||
}
|
||||
launcher := strings.TrimSuffix(entry, ".js") + launcherSuffix
|
||||
body := "#!/usr/bin/env node\n" +
|
||||
"// Written by the mesh's builder (novox/hq ADR 0193): serve what " + entry + " registers,\n" +
|
||||
"// over MCP on stdio, as the module the node's runtime names in MESH_SERVED_MODULE.\n" +
|
||||
"import { serveRegisteredOverStdio } from \"@novox/mesh-sdk/stdio\";\n" +
|
||||
"await import(\"./" + filepath.Base(entry) + "\");\n" +
|
||||
"await serveRegisteredOverStdio();\n"
|
||||
path := filepath.Join(root, filepath.FromSlash(launcher))
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(path, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[entry] = launcher
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sourcesFor turns compiled entrypoints back into what to compile (builder.go:1735-1747).
|
||||
func sourcesFor(entrypoints []string, out, ext string) []string {
|
||||
sources := make([]string, 0, len(entrypoints))
|
||||
for _, e := range entrypoints {
|
||||
at := strings.TrimPrefix(strings.TrimPrefix(e, out), "/")
|
||||
sources = append(sources, strings.TrimSuffix(at, filepath.Ext(at))+ext)
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
func sortedValues(m map[string]string) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for _, v := range m {
|
||||
out = append(out, v)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// readArtifact reads the named artifact of the module.json in dir, which must be a TypeScript bundle with entrypoints.
|
||||
func readArtifact(dir, name string) (artifact, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return artifact{}, fmt.Errorf("the module has no module.json: %w", err)
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return artifact{}, fmt.Errorf("module.json is not JSON: %w", err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Name != name {
|
||||
continue
|
||||
}
|
||||
if a.Language != "typescript" {
|
||||
return artifact{}, fmt.Errorf("artifact %q is %q, and this module builds typescript", name, a.Language)
|
||||
}
|
||||
if len(a.Entrypoints) == 0 {
|
||||
return artifact{}, fmt.Errorf("artifact %q names no entrypoints", name)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
return artifact{}, fmt.Errorf("module.json declares no artifact %q", name)
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
type npmCall struct {
|
||||
name string
|
||||
args []string
|
||||
env []string
|
||||
}
|
||||
|
||||
type fakeNpm struct {
|
||||
mu sync.Mutex
|
||||
calls []npmCall
|
||||
ran forgekey.Ran
|
||||
}
|
||||
|
||||
func (f *fakeNpm) run(_ context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, npmCall{name, args, env})
|
||||
if f.ran.Status == 0 && f.ran.Err == "" {
|
||||
// npm ci writes node_modules into --prefix.
|
||||
for i, a := range args {
|
||||
if a == "--prefix" {
|
||||
os.MkdirAll(filepath.Join(args[i+1], "node_modules"), 0o700)
|
||||
}
|
||||
}
|
||||
}
|
||||
return f.ran
|
||||
}
|
||||
|
||||
func toolchainIn(t *testing.T, fake *fakeNpm) Toolchain {
|
||||
t.Helper()
|
||||
home, cache := t.TempDir(), t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(home, "toolchain"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(home, "toolchain", "package.json"), []byte(`{"name":"x"}`), 0o600)
|
||||
os.WriteFile(filepath.Join(home, "toolchain", "package-lock.json"), []byte(`{"lock":1}`), 0o600)
|
||||
return Toolchain{Paths: Paths{Home: home, Cache: cache}, Run: fake.run,
|
||||
Registry: func() (Registry, error) {
|
||||
return Registry{URL: "https://git.novox.be/api/packages/novox/npm/", Username: "mesh-build-node", Password: "pw"}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// 1488/AC4: given a missing toolchain directory, npm ci --ignore-scripts runs once in <cache>/toolchain/<recipe
|
||||
// version> with the module's lock; asked again with the directory present, nothing runs.
|
||||
func TestAMissingToolchainDirectoryIsInstalledOnceFromTheLock(t *testing.T) {
|
||||
fake := &fakeNpm{}
|
||||
tc := toolchainIn(t, fake)
|
||||
version := Recipe{Steps: "1", Node: "v26.10.0", Lock: "x"}.Version()
|
||||
dir := filepath.Join(tc.Paths.Cache, "toolchain", version)
|
||||
if tc.Present(version) {
|
||||
t.Fatal("present before any install")
|
||||
}
|
||||
if err := tc.Ensure(context.Background(), version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fake.calls) != 1 {
|
||||
t.Fatalf("npm ran %d times, want once", len(fake.calls))
|
||||
}
|
||||
c := fake.calls[0]
|
||||
joined := strings.Join(c.args, " ")
|
||||
if c.name != "npm" || c.args[0] != "ci" || !strings.Contains(joined, "--ignore-scripts") || !strings.Contains(joined, "--no-bin-links") || !strings.Contains(joined, "--prefix "+dir) {
|
||||
t.Errorf("ran %s %v, want npm ci --ignore-scripts --prefix %s", c.name, c.args, dir)
|
||||
}
|
||||
if lock, _ := os.ReadFile(filepath.Join(dir, "package-lock.json")); string(lock) != `{"lock":1}` {
|
||||
t.Errorf("the directory holds lock %q, want the module's", lock)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, ".npmrc-install")); err == nil {
|
||||
t.Errorf("the registry credential file was left in the toolchain directory")
|
||||
}
|
||||
if !tc.Present(version) {
|
||||
t.Errorf("not present after the install")
|
||||
}
|
||||
if err := tc.Ensure(context.Background(), version); err != nil || len(fake.calls) != 1 {
|
||||
t.Errorf("a present directory ran npm again: %d calls, %v", len(fake.calls), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC4: the install is told the mesh's registry for @novox with the account's credential, and only through a
|
||||
// config file of its own, never on the command line.
|
||||
func TestTheInstallNamesTheRegistryInAConfigFileNotOnTheCommandLine(t *testing.T) {
|
||||
var npmrc string
|
||||
fake := &fakeNpm{}
|
||||
tc := toolchainIn(t, fake)
|
||||
tc.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
for i, a := range args {
|
||||
if a == "--userconfig" {
|
||||
raw, _ := os.ReadFile(args[i+1])
|
||||
npmrc = string(raw)
|
||||
}
|
||||
}
|
||||
return fake.run(ctx, env, name, args...)
|
||||
}
|
||||
if err := tc.Ensure(context.Background(), "v1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(npmrc, "@novox:registry=https://git.novox.be/api/packages/novox/npm/\n") ||
|
||||
!strings.Contains(npmrc, "//git.novox.be/api/packages/novox/npm/:_auth=") {
|
||||
t.Errorf("npmrc = %q", npmrc)
|
||||
}
|
||||
if strings.Contains(strings.Join(fake.calls[0].args, " "), "pw") {
|
||||
t.Errorf("the credential is on the command line: %v", fake.calls[0].args)
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC4: an install that fails leaves no directory taken for a toolchain, says why, and the next call tries again.
|
||||
func TestAFailedInstallIsNotAToolchain(t *testing.T) {
|
||||
fake := &fakeNpm{ran: forgekey.Ran{Status: 1, Stderr: "E401 unauthorized"}}
|
||||
tc := toolchainIn(t, fake)
|
||||
err := tc.Ensure(context.Background(), "v1")
|
||||
if err == nil || !strings.Contains(err.Error(), "E401") {
|
||||
t.Fatalf("err = %v, want the npm failure named", err)
|
||||
}
|
||||
if tc.Present("v1") {
|
||||
t.Errorf("a failed install is present")
|
||||
}
|
||||
fake.ran = forgekey.Ran{}
|
||||
if err := tc.Ensure(context.Background(), "v1"); err != nil || len(fake.calls) != 2 || !tc.Present("v1") {
|
||||
t.Errorf("the retry: %v, %d calls", err, len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
// 1488/AC4: a module's steps run the install behind the guard: a refusing node installs nothing, an accepting one
|
||||
// installs once however many steps follow.
|
||||
func TestPrepareInstallsOnlyBehindAPassingGuard(t *testing.T) {
|
||||
fake := &fakeNpm{}
|
||||
tc := toolchainIn(t, fake)
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.11.0")
|
||||
m.Toolchain = &tc
|
||||
if err := m.Prepare(context.Background()); err == nil || len(fake.calls) != 0 {
|
||||
t.Fatalf("a refusing node: %v, %d npm calls", err, len(fake.calls))
|
||||
}
|
||||
m = moduleWith(states, "v26.10.0")
|
||||
m.Toolchain = &tc
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := m.Prepare(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(fake.calls) != 1 {
|
||||
t.Errorf("npm ran %d times for three steps, want once", len(fake.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFromTheBinding(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b, s := filepath.Join(dir, "b.json"), filepath.Join(dir, "s")
|
||||
os.WriteFile(s, []byte("secret\n"), 0o600)
|
||||
os.WriteFile(b, []byte(`{"provision":"npm-package-registry","at":"10.42.0.1","as":"mesh-build-node-ace","serves":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/"}}`), 0o600)
|
||||
r, err := ReadRegistry(b, s)
|
||||
if err != nil || r.URL != "http://10.42.0.1:3000/api/packages/novox/npm/" || r.Username != "mesh-build-node-ace" || r.Password != "secret" {
|
||||
t.Fatalf("= %+v, %v", r, err)
|
||||
}
|
||||
os.WriteFile(b, []byte(`{"at":"h","as":"a","serves":{"port":1,"npm-path":"/p/","public-root":"https://git.novox.be/p"}}`), 0o600)
|
||||
if r, _ := ReadRegistry(b, s); r.URL != "https://git.novox.be/p/" {
|
||||
t.Errorf("the public root is not preferred: %q", r.URL)
|
||||
}
|
||||
if _, err := ReadRegistry(filepath.Join(dir, "none"), s); err == nil {
|
||||
t.Errorf("no binding was accepted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user