Files
mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/package.go
T
jschoubben 6476e3595a Node build module: install what a placed mesh package declares (H-A-491)
A mesh package's tarball is unpacked straight into node_modules, so its own
dependencies were never there: the forge module's bundle failed with esbuild
"Could not resolve blakejs" from the SDK's seal code. After every mesh package
is placed, the fetch installs the dependencies they declare (not their
devDependencies, not a mesh package, not one already present) through the
same npm path as the module's own, before the bundle.

Builds: design 47 H H-A-491 (addendum)
2026-11-22 20:03:48 +01:00

956 lines
33 KiB
Go

package main
// A TypeScript package (novox/hq ADR 0498 §5, ADR 0500 §4-§6; issue 1587): built from its own lock's devDependencies
// (the toolchain's SDK is never used to build the SDK itself), packed with npm pack, and, in a group's check, served to
// the fetch of every member: each package the group changes is packed and placed by its package.json name in every
// module that names it (the SDK in every module), in place of the registry's (design 47 G A-284, A-285); a package the
// group does not change comes from the registry at its latest published version (ADR 0573 §2), the SDK's a wait
// (`waits for:`) while the registry holds none. The install of a package's devDependencies is the fetch's (it holds the
// network and runs no script); its build and pack run shut off like every run step.
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// maxPackage is the largest packed package read: far above the SDK's.
const maxPackage = 64 << 20
// packed is a package built and packed: its name and version from its package.json, the packed file's bytes, its npm
// integrity (sha512, as a lock names it) and its digest (sha256, as a done key names it).
type packed struct {
Name, Version, Integrity, Digest string
tgz []byte
}
// integrityOf is a packed file's integrity as npm writes it in a lock: sha512- and the base64 of its sha512.
func integrityOf(b []byte) string {
sum := sha512.Sum512(b)
return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
}
func digestOf(b []byte) string {
sum := sha256.Sum256(b)
return "sha256:" + hex.EncodeToString(sum[:])
}
// packageJSON is the part of a package.json the package steps read.
type packageJSON struct {
Name string `json:"name"`
Version string `json:"version"`
Scripts map[string]string `json:"scripts"`
}
func readPackageJSON(dir string) (packageJSON, error) {
var p packageJSON
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if err != nil {
return p, fmt.Errorf("the package has no package.json: %w", err)
}
if err := json.Unmarshal(raw, &p); err != nil {
return p, fmt.Errorf("the package.json is not JSON: %w", err)
}
if p.Name == "" || p.Version == "" {
return p, fmt.Errorf("the package.json names no name or no version")
}
return p, nil
}
// installDev installs a package's own dependencies, its devDependencies included, from its lock (the fetch: network,
// no script, no bin link), and writes a shim for each program they carry into <root>/bin-<name>, files and not links,
// so its build script finds them by name.
func (m *Module) installDev(ctx context.Context, root, dir, rel string) error {
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
return fmt.Errorf("the package in %s has no package-lock.json: commit one", rel)
}
npmrc, err := m.npmrcIn(root)
if err != nil {
return err
}
defer os.Remove(npmrc)
env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
r := m.Run(ctx, env, "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc)
if why := ranFailed(r); why != "" {
return steprun.Transport(fmt.Errorf("installing the dependencies of the package in %s: %s", rel, why))
}
return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel))
}
// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the
// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4).
func (m *Module) npmrcIn(root string) (string, error) {
if m.Toolchain == nil || m.Toolchain.Registry == nil {
return "", fmt.Errorf("the module has no package registry")
}
reg, err := m.Toolchain.Registry()
if err != nil {
return "", err
}
f, err := os.CreateTemp("", ".npmrc-")
if err != nil {
return "", err
}
defer f.Close()
if _, err := f.WriteString(reg.Npmrc()); err != nil {
os.Remove(f.Name())
return "", err
}
return f.Name(), nil
}
// shimDir is where the shims of the package in rel are written.
func shimDir(root, rel string) string {
return filepath.Join(root, "bin-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
}
// writeShims writes, for every program a top-level package of nm names in its package.json `bin`, a shell script that
// runs it by its real path.
func writeShims(nm, into string) error {
if err := os.MkdirAll(into, 0o755); err != nil {
return err
}
var dirs []string
entries, _ := os.ReadDir(nm)
for _, e := range entries {
if !e.IsDir() || strings.HasPrefix(e.Name(), ".") {
continue
}
if strings.HasPrefix(e.Name(), "@") {
scoped, _ := os.ReadDir(filepath.Join(nm, e.Name()))
for _, s := range scoped {
if s.IsDir() {
dirs = append(dirs, filepath.Join(nm, e.Name(), s.Name()))
}
}
continue
}
dirs = append(dirs, filepath.Join(nm, e.Name()))
}
sort.Strings(dirs)
for _, d := range dirs {
raw, err := os.ReadFile(filepath.Join(d, "package.json"))
if err != nil {
continue
}
var p struct {
Name string `json:"name"`
Bin json.RawMessage `json:"bin"`
}
if json.Unmarshal(raw, &p) != nil || len(p.Bin) == 0 {
continue
}
bins := map[string]string{}
var one string
if json.Unmarshal(p.Bin, &one) == nil {
bins[path.Base(p.Name)] = one
} else if json.Unmarshal(p.Bin, &bins) != nil {
continue
}
for name, target := range bins {
if name == "" || strings.ContainsAny(name, "/\\") {
continue
}
body := "#!/bin/sh\nexec node '" + filepath.Join(d, filepath.FromSlash(target)) + "' \"$@\"\n"
p := filepath.Join(into, name)
if err := os.WriteFile(p, []byte(body), 0o755); err != nil {
return err
}
if err := os.Chmod(p, 0o755); err != nil {
return err
}
}
}
return nil
}
// buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both
// shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never
// stays on the node.
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) {
if version != "" {
restore, err := setPackageVersion(dir, version)
if err != nil {
return packed{}, err
}
defer restore()
}
p, err := readPackageJSON(dir)
if err != nil {
return packed{}, err
}
env := []string{"PATH=" + shimDir(root, rel) + ":" + pathOrDefault(), "npm_config_cache=" + m.npmCache()}
env = append(env, versionEnv(version)...)
if p.Scripts["build"] != "" {
if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "run", "build")); why != "" {
return packed{}, fmt.Errorf("the build of the package %s: %s", p.Name, why)
}
}
out, err := os.MkdirTemp(root, "packed-")
if err != nil {
return packed{}, err
}
defer os.RemoveAll(out)
if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "pack", "--ignore-scripts", "--pack-destination", out)); why != "" {
return packed{}, fmt.Errorf("packing the package %s: %s", p.Name, why)
}
files, _ := filepath.Glob(filepath.Join(out, "*.tgz"))
if len(files) != 1 {
return packed{}, fmt.Errorf("npm pack of %s left %d packed files, want one", p.Name, len(files))
}
tgz, err := os.ReadFile(files[0])
if err != nil {
return packed{}, err
}
return packed{Name: p.Name, Version: p.Version, Integrity: integrityOf(tgz), Digest: digestOf(tgz), tgz: tgz}, nil
}
// groupPackage is one entry of the fetch's `group-packages` argument (design 47 G A-284, A-285; ADR 0500 §5): a
// TypeScript package a member of the group changes, at that member's commit, with Path its directory in the member (""
// for the member's root). The fetch packs it, publishing nothing, and places it by its package.json name in every module
// of the tree that names it, in place of the registry's.
type groupPackage struct{ Repository, Commit, Ref, Path string }
// groupPackagesArg reads the fetch's `group-packages`: a list of {repository, commit, ref, path}; anything else is
// refused before anything is fetched.
func groupPackagesArg(v any) ([]groupPackage, error) {
if v == nil {
return nil, nil
}
list, ok := v.([]any)
if !ok {
return nil, fmt.Errorf("group-packages is a list of {repository, commit, ref, path}")
}
var out []groupPackage
for _, e := range list {
o, ok := e.(map[string]any)
if !ok {
return nil, fmt.Errorf("group-packages: %v is not {repository, commit, ref, path}", e)
}
var g groupPackage
g.Repository, _ = o["repository"].(string)
g.Commit, _ = o["commit"].(string)
g.Ref, _ = o["ref"].(string)
g.Path, _ = o["path"].(string)
if !repositoryOK.MatchString(g.Repository) || strings.Contains(g.Repository, "..") {
return nil, fmt.Errorf("group-packages: %q is not a repository, <owner>/<name>", g.Repository)
}
if !commitOK.MatchString(g.Commit) {
return nil, fmt.Errorf("group-packages: %q is not a full commit (40 hex digits)", g.Commit)
}
if g.Ref != "" && !strings.HasPrefix(g.Ref, "refs/pull/") {
return nil, fmt.Errorf("group-packages: %q is not a pull request's head", g.Ref)
}
if g.Path != "" {
c := path.Clean(g.Path)
if path.IsAbs(c) || c == ".." || strings.HasPrefix(c, "../") || strings.Contains(g.Path, "\\") {
return nil, fmt.Errorf("group-packages: %q is not a directory inside the member", g.Path)
}
if c == "." {
c = ""
}
g.Path = c
}
out = append(out, g)
}
return out, nil
}
// namedPackages are the packages a module's package.json names in any of its dependency lists; none without one.
func namedPackages(dir string) map[string]bool {
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if err != nil {
return nil
}
var p map[string]any
if json.Unmarshal(raw, &p) != nil {
return nil
}
out := map[string]bool{}
for _, k := range []string{"dependencies", "devDependencies", "peerDependencies", "optionalDependencies"} {
deps, _ := p[k].(map[string]any)
for name := range deps {
out[name] = true
}
}
return out
}
// packGroups fetches each of the group's packages into the workspace, once per member commit, and, when a module of the
// tree takes it (wanted, by its package.json name), installs its dependencies, builds and packs it, publishing nothing
// (ADR 0500 §4, §5). A package no module takes is not built. Answers the packed packages by name.
func (m *Module) packGroups(ctx context.Context, root string, run forgekey.Runner, gs []groupPackage, wanted func(name string) bool) (map[string]*packed, error) {
served := map[string]*packed{}
trees := map[string]string{}
for i, g := range gs {
key := g.Repository + "@" + g.Commit
tree, ok := trees[key]
if !ok {
tree = filepath.Join(root, "group", strconv.Itoa(len(trees)), "tree")
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
if err := fetchCommit(ctx, run, tree, m.remote(g.Repository), g.Commit, g.Ref, m); err != nil {
return nil, fmt.Errorf("the group's package %s at %s: %w", g.Repository, g.Commit, err)
}
trees[key] = tree
}
dir := filepath.Join(tree, filepath.FromSlash(g.Path))
p, err := readPackageJSON(dir)
if err != nil {
return nil, fmt.Errorf("the group's package %s %q: %w", g.Repository, g.Path, err)
}
if served[p.Name] != nil {
return nil, fmt.Errorf("the group names two packages %s", p.Name)
}
if !wanted(p.Name) {
continue
}
rel := "group-" + strconv.Itoa(i)
if err := m.installDev(ctx, root, dir, rel); err != nil {
return nil, err
}
pk, err := m.buildAndPack(ctx, root, dir, rel, false, "")
if err != nil {
return nil, err
}
served[pk.Name] = &pk
}
return served, nil
}
// npmVersionOK is a semantic version as npm names one, with an optional pre-release.
var npmVersionOK = regexp.MustCompile(`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
// placeGroup places in the module's node_modules each of the group's packed packages its package.json names, the SDK
// aside (placeSDK places it), in place of whatever the install put there.
func placeGroup(dir string, served map[string]*packed) error {
named := namedPackages(dir)
self, _ := readPackageJSON(dir)
for name, p := range served {
if name == sdkPackage || !named[name] || name == self.Name {
continue
}
into := filepath.Join(dir, "node_modules", filepath.FromSlash(name))
if err := os.RemoveAll(into); err != nil {
return err
}
if err := unpackPackage(p.tgz, into); err != nil {
return err
}
}
return nil
}
// locked is what a module's lock names of the package name: its version and its integrity; ok is false when the lock
// names none (the latest release is then placed, placeLatest).
func locked(dir, name string) (version, integrity string, ok bool, err error) {
raw, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
if os.IsNotExist(err) {
return "", "", false, nil
}
if err != nil {
return "", "", false, err
}
var lock struct {
Packages map[string]struct {
Version string `json:"version"`
Integrity string `json:"integrity"`
} `json:"packages"`
}
if err := json.Unmarshal(raw, &lock); err != nil {
return "", "", false, fmt.Errorf("the package-lock.json is not JSON: %w", err)
}
p, ok := lock.Packages["node_modules/"+name]
if !ok || p.Version == "" {
return "", "", false, nil
}
return p.Version, p.Integrity, true, nil
}
// placeSDK places in the module's node_modules the SDK it is built against: the latest release the registry holds,
// whatever its lock names (novox/hq ADR 0573 §2: a module builds against the latest published version of each mesh
// package; the lead's decision L10's version from the lock is gone), or, when the group changes the SDK, the group's
// packed package in place of the registry's, whatever the lock names (design 47 G A-284, A-285). Every TypeScript module
// gets it, named in its package.json or not, since its launcher imports the SDK (writeLaunchers) and the toolchain holds
// none. The lock's integrity is checked when the lock names the registry's version placed; a registry holding no
// release is a wait (`waits for:`).
func (m *Module) placeSDK(ctx context.Context, dir, rel string, group *packed) error {
if group == nil {
return m.placeLatest(ctx, dir, rel, sdkPackage)
}
version, _, _, err := locked(dir, sdkPackage)
if err != nil {
return err
}
if m.Say != nil && version != "" && version != group.Version {
m.Say("%s: its lock names %s@%s; the group's %s is placed (design 47 G A-284)", rel, sdkPackage, version, group.Version)
}
into := filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage))
if err := os.RemoveAll(into); err != nil {
return err
}
return unpackPackage(group.tgz, into)
}
// placeMeshPackages places in the module's node_modules each mesh package beyond the SDK (@novox/mesh-*) its
// package.json names at the latest release the registry holds, as placeSDK does for the SDK (ADR 0573 §2; design 47 H,
// H-A-267): the install leaves them out (ownDependencies). One the group serves or a member's build lays is left to
// placeGroup and layBuilds, which take precedence; the module itself is not placed.
func (m *Module) placeMeshPackages(ctx context.Context, dir, rel string, served map[string]*packed, members []storedBuild) error {
self, _ := readPackageJSON(dir)
var names []string
for name := range namedPackages(dir) {
if name == sdkPackage || !strings.HasPrefix(name, "@novox/mesh-") || name == self.Name || served[name] != nil || namesPackage(members, name) {
continue
}
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
if err := m.placeLatest(ctx, dir, rel, name); err != nil {
return err
}
}
return nil
}
// placeLatest places in the module's node_modules the latest release of the mesh package name the registry holds,
// whatever the module's lock names (ADR 0573 §2), checked against the lock's integrity when the lock names that
// version; a registry holding no release is a wait (`waits for:`).
func (m *Module) placeLatest(ctx context.Context, dir, rel, name string) error {
version, integrity, _, err := locked(dir, name)
if err != nil {
return err
}
latest, err := m.registryLatest(ctx, name)
if err != nil {
return err
}
if latest == "" {
return fmt.Errorf("%s%s", builds.WaitsFor, name)
}
if version != latest {
if m.Say != nil && version != "" {
m.Say("%s: its lock names %s@%s; the latest published %s is placed (ADR 0573 §2)", rel, name, version, latest)
}
version, integrity = latest, ""
}
if spec := namedRange(dir, name); spec != "" && m.Say != nil {
// Said, never refused: the latest release is placed whatever the range (ADR 0573 §2; design 47 H, H-A-276).
switch in, read := inRange(spec, version); {
case !read:
m.Say("%s: its package.json names %s at %q, a range not read; the latest published %s is placed (ADR 0573 §2)", rel, name, spec, version)
case !in:
m.Say("%s: the latest published %s@%s lies outside the range %q its package.json names; it is placed (ADR 0573 §2)", rel, name, version, spec)
}
}
want := name + "@" + version
got, registered, held, err := m.registryPackageChecked(ctx, name, version)
if err != nil {
return err
}
if !held {
return fmt.Errorf("%s%s", builds.WaitsFor, want)
}
if integrity == "" {
integrity = registered
}
if integrity != "" && integrityOf(got) != integrity {
return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity)
}
into := filepath.Join(dir, "node_modules", filepath.FromSlash(name))
if err := os.RemoveAll(into); err != nil {
return err
}
return unpackPackage(got, into)
}
// namedRange is the range the module's package.json names of name: its dependencies first, then its peer, optional and
// dev dependencies; "" when it names none.
func namedRange(dir, name string) string {
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
if err != nil {
return ""
}
var p map[string]any
if json.Unmarshal(raw, &p) != nil {
return ""
}
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies", "devDependencies"} {
deps, _ := p[k].(map[string]any)
if spec, ok := deps[name].(string); ok {
return strings.TrimSpace(spec)
}
}
return ""
}
// inRange says whether the release v (major.minor.patch) lies in the npm range spec; read is false for a range of a
// form it does not read (a union, a hyphen range, a comparator, a tag other than latest), never taken as in or out. It
// reads what a package file of the mesh names: an exact version, an x-range, a caret and a tilde range, as npm does.
func inRange(spec, v string) (in, read bool) {
spec = strings.TrimSpace(spec)
if spec == "" || spec == "*" || spec == "latest" {
return true, true
}
if strings.ContainsAny(spec, " |<>") {
return false, false
}
op := ""
if spec[0] == '^' || spec[0] == '~' {
op, spec = spec[:1], spec[1:]
}
spec = strings.TrimPrefix(strings.TrimPrefix(spec, "="), "v")
if strings.Contains(spec, "-") || strings.Contains(spec, "+") {
return false, false // a pre-release or build range
}
var low [3]int
given := 0
for i, part := range strings.Split(spec, ".") {
if i > 2 {
return false, false
}
if part == "x" || part == "X" || part == "*" {
break
}
n, err := strconv.Atoi(part)
if err != nil || n < 0 {
return false, false
}
low[i] = n
given = i + 1
}
if given == 0 {
return true, true // x, x.x
}
var at [3]int
for i, part := range strings.Split(v, ".") {
if i > 2 {
return false, false
}
n, err := strconv.Atoi(part)
if err != nil {
return false, false
}
at[i] = n
}
// bump is the part of low raised by one for the exclusive upper bound, the parts after it zero.
bump := given - 1
switch op {
case "^":
switch {
case low[0] > 0 || given == 1:
bump = 0
case low[1] > 0 || given == 2:
bump = 1
default:
bump = 2
}
case "~":
bump = 0
if given > 1 {
bump = 1
}
}
high := low
high[bump]++
for i := bump + 1; i < 3; i++ {
high[i] = 0
}
less := func(a, b [3]int) bool {
for i := range a {
if a[i] != b[i] {
return a[i] < b[i]
}
}
return false
}
return !less(at, low) && less(at, high), true
}
// placedSDK is the version of the SDK placed in the module's node_modules ("" when none is).
func placedSDK(dir string) string {
p, err := readPackageJSON(filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage)))
if err != nil {
return ""
}
return p.Version
}
// registryLatest is the latest release of name the registry holds: the highest version with no pre-release, so a
// group's tagged version is never taken for it; "" when the registry holds none.
func (m *Module) registryLatest(ctx context.Context, name string) (string, error) {
p, held, err := m.packumentOf(ctx, name)
if err != nil || !held {
return "", err
}
latest := ""
for v := range p.Versions {
if !npmVersionOK.MatchString(v) || strings.Contains(v, "-") {
continue
}
if latest == "" || releaseLess(latest, v) {
latest = v
}
}
return latest, nil
}
// releaseLess orders two releases (major.minor.patch, no pre-release) by their numbers.
func releaseLess(a, b string) bool {
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
for i := 0; i < 3; i++ {
x, _ := strconv.Atoi(pa[i])
y, _ := strconv.Atoi(pb[i])
if x != y {
return x < y
}
}
return false
}
// registryPackageChecked is registryPackage with the integrity the registry's document names of name@version.
func (m *Module) registryPackageChecked(ctx context.Context, name, version string) (tgz []byte, integrity string, held bool, err error) {
_, integrity, held, err = m.registryVersion(ctx, name, version)
if err != nil || !held {
return nil, "", held, err
}
tgz, held, err = m.registryPackage(ctx, name, version)
return tgz, integrity, held, err
}
// httpClient is the client of the registry's reads and publishes: HTTP when a test set it.
func (m *Module) httpClient() *http.Client {
if m.HTTP != nil {
return m.HTTP
}
return &http.Client{Timeout: 5 * time.Minute}
}
// packument is the part of the registry's document of a package the steps read: each version's packed file.
type packument struct {
Versions map[string]struct {
Dist struct {
Tarball string `json:"tarball"`
Integrity string `json:"integrity"`
} `json:"dist"`
} `json:"versions"`
}
// packumentOf reads the registry's document of name; held is false when the registry has none.
func (m *Module) packumentOf(ctx context.Context, name string) (packument, bool, error) {
reg, err := m.Toolchain.Registry()
if err != nil {
return packument{}, false, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reg.URL+url.PathEscape(name), nil)
if err != nil {
return packument{}, false, err
}
resp, err := m.httpClient().Do(req)
if err != nil {
return packument{}, false, fmt.Errorf("asking the registry for %s: %w", name, err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return packument{}, false, nil
}
if resp.StatusCode != http.StatusOK {
said, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return packument{}, false, fmt.Errorf("the registry answered %s for %s: %s", resp.Status, name, strings.TrimSpace(string(said)))
}
var p packument
if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<20)).Decode(&p); err != nil {
return packument{}, false, fmt.Errorf("the registry's document of %s: %w", name, err)
}
return p, true, nil
}
// registryVersion reads what the registry holds of name@version: its packed file's address and integrity, and whether
// it holds it at all.
func (m *Module) registryVersion(ctx context.Context, name, version string) (tarball, integrity string, held bool, err error) {
p, held, err := m.packumentOf(ctx, name)
if err != nil || !held {
return "", "", false, err
}
v, ok := p.Versions[version]
if !ok {
return "", "", false, nil
}
return v.Dist.Tarball, v.Dist.Integrity, true, nil
}
// registryPackage downloads name@version's packed file from the registry; held is false when the registry has none.
func (m *Module) registryPackage(ctx context.Context, name, version string) ([]byte, bool, error) {
tarball, _, held, err := m.registryVersion(ctx, name, version)
if err != nil || !held {
return nil, held, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil)
if err != nil {
return nil, false, err
}
resp, err := m.httpClient().Do(req)
if err != nil {
return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("the registry answered %s for %s@%s's packed file", resp.Status, name, version)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxPackage+1))
if err != nil {
return nil, false, err
}
if len(body) > maxPackage {
return nil, false, fmt.Errorf("%s@%s is larger than %d bytes", name, version, maxPackage)
}
return body, true, nil
}
// unpackPackage unpacks a packed npm package (a gzipped tar whose entries lie under package/) into dir: files and
// directories only, nothing outside.
func unpackPackage(tgz []byte, dir string) error {
zr, err := gzip.NewReader(bytes.NewReader(tgz))
if err != nil {
return fmt.Errorf("the packed package is not gzip: %w", err)
}
tr := tar.NewReader(zr)
for {
h, err := tr.Next()
if err == io.EOF {
return nil
}
if err != nil {
return err
}
name := path.Clean(strings.TrimPrefix(h.Name, "./"))
rel, ok := strings.CutPrefix(name, "package/")
if !ok || rel == "" || path.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, "../") {
if name == "package" {
continue
}
return fmt.Errorf("the packed package holds %s, outside package/", h.Name)
}
target := filepath.Join(dir, filepath.FromSlash(rel))
switch h.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
body, err := io.ReadAll(io.LimitReader(tr, maxPackage))
if err != nil {
return err
}
if err := os.WriteFile(target, body, os.FileMode(h.Mode).Perm()|0o600); err != nil {
return err
}
default:
return fmt.Errorf("the packed package holds %s, neither a file nor a directory", h.Name)
}
}
}
// fetchCommit fetches one commit into dir as the fetch step fetches the module's own (steprun.FetchCommit).
func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, m *Module) error {
return steprun.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...)
}
// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build
// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry
// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks
// the builder, answers held for the same file and refuses another under that version. The reference is
// <name>@<version>, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its
// digest alone. dir is the package's directory, at its place in the module's.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir, at string, a artifact, publish bool, version, seat string) (any, error) {
// rel is the package's directory in the workspace, as the fetch installed it (the shims lie under it).
rel := path.Join(module, at)
if module == "" {
rel = path.Join(".", at)
}
if !publish {
version = ""
}
if seat == "" {
seat = steprun.NpmSeat
}
if version != "" && seat != steprun.NpmSeat {
return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, steprun.NpmSeat, seat)
}
p, err := m.buildAndPack(ctx, root, dir, rel, publish, version)
if err != nil {
return nil, err
}
res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)}
if version == "" {
return res, nil
}
if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil {
return nil, fmt.Errorf("%s%s: %w", builds.NotStored, a.Name, err)
}
res.Reference = p.Name + "@" + p.Version
return res, nil
}
// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm
// packs the package under it; restore puts the file back as it was.
func setPackageVersion(dir, version string) (restore func(), err error) {
path := filepath.Join(dir, "package.json")
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("the package's package.json: %w", err)
}
var doc map[string]json.RawMessage
if err := json.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("the package's package.json does not parse: %w", err)
}
doc["version"], _ = json.Marshal(version)
out, err := json.MarshalIndent(doc, "", " ")
if err != nil {
return nil, err
}
if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil {
return nil, err
}
return func() { _ = os.WriteFile(path, raw, 0o644) }, nil
}
// installPlacedDependencies installs into the module's node_modules the dependencies every mesh package placed there
// (@novox/mesh-*: placeSDK, placeMeshPackages, placeGroup, layBuilds) declares, as a registry install of that package
// would (design 47 H, H-A-491 addendum): a placed package's tarball is unpacked whole, and its built code imports what
// its package.json's `dependencies` name (the SDK's seal code imports blakejs). Its devDependencies are left out, a mesh
// package and what is build-time are not installed (ownDependencies), nor one the module's node_modules holds already.
// The install is installOwn's: a scratch package.json naming each at the range its package declares (the first placed
// package by name decides a name two declare), `npm install` through the registry's config file and the npm cache,
// the install's copies of the SDK removed, and each package it put that the module's node_modules lacks moved there.
func (m *Module) installPlacedDependencies(ctx context.Context, root, dir, rel string) error {
nm := filepath.Join(dir, "node_modules")
entries, err := os.ReadDir(filepath.Join(nm, "@novox"))
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
wanted := map[string]any{}
for _, e := range entries {
if !e.IsDir() || !strings.HasPrefix(e.Name(), "mesh-") {
continue
}
names, pkg, err := ownDependencies(filepath.Join(nm, "@novox", e.Name()))
if err != nil {
return fmt.Errorf("the placed @novox/%s: %w", e.Name(), err)
}
deps, _ := pkg["dependencies"].(map[string]any)
for _, name := range names {
if _, ok := wanted[name]; ok {
continue
}
if _, err := os.Stat(filepath.Join(nm, filepath.FromSlash(name), "package.json")); err == nil {
continue
}
wanted[name] = deps[name]
}
}
if len(wanted) == 0 {
return nil
}
var names []string
for name := range wanted {
names = append(names, name)
}
sort.Strings(names)
scratch := filepath.Join(root, "npm-placed-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
if err := os.MkdirAll(scratch, 0o700); err != nil {
return err
}
defer os.RemoveAll(scratch)
raw, _ := json.Marshal(map[string]any{"name": "placed-dependencies", "private": true, "dependencies": wanted})
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
return err
}
npmrc, err := m.npmrcIn(root)
if err != nil {
return fmt.Errorf("installing the dependencies of the mesh packages placed in %s: %w", rel, err)
}
env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
r := m.Run(ctx, env, "npm", "install", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
os.Remove(npmrc)
if why := ranFailed(r); why != "" {
return steprun.Transport(fmt.Errorf("installing the dependencies of the mesh packages placed in %s (%s): %s", rel, strings.Join(names, ", "), why))
}
got := filepath.Join(scratch, "node_modules")
if err := removeSDKCopies(got); err != nil {
return err
}
if err := moveAbsent(got, nm); err != nil {
return err
}
if m.Say != nil {
m.Say("%s: installed %s, which the mesh packages placed there declare (design 47 H, H-A-491)", rel, strings.Join(names, ", "))
}
return nil
}
// moveAbsent moves each package of the node_modules from (a scope's packages one by one) to the node_modules into
// that holds none of its name; npm's own files (.bin, .package-lock.json) stay.
func moveAbsent(from, into string) error {
entries, err := os.ReadDir(from)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") || !e.IsDir() {
continue
}
if strings.HasPrefix(e.Name(), "@") {
if err := moveAbsent(filepath.Join(from, e.Name()), filepath.Join(into, e.Name())); err != nil {
return err
}
continue
}
target := filepath.Join(into, e.Name())
if _, err := os.Stat(target); err == nil {
continue
}
if err := os.MkdirAll(into, 0o755); err != nil {
return err
}
if err := os.Rename(filepath.Join(from, e.Name()), target); err != nil {
return err
}
}
return nil
}