mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/package.go

package main

// A TypeScript package (novox/hq ADR 0498 §5, ADR 0500 §4-§6; issue 1587, the lead's decision L10): built from its own
// lock's devDependencies (the toolchain's SDK is never used to build the SDK itself), packed with npm pack, and, for a
// group, served to its TypeScript members' fetch in place of the registry's version. A member is built against the SDK
// version its own lock names: the fetch places that version in the member's node_modules, from the group's packed
// package when the group carries that version, else from the registry; a version neither holds waits for its
// publication (`waits for:`). The install of a package's devDependencies is the fetch's (it holds the network and runs
// no script); its build and pack run shut off like every run step.

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// maxPackage is the largest packed package read: far above the SDK's.
const maxPackage = 64 << 20

// packed is a package built and packed: its name and version from its package.json, the packed file's bytes, its npm
// integrity (sha512, as a lock names it) and its digest (sha256, as a done key names it).
type packed struct {
	Name, Version, Integrity, Digest string
	tgz                              []byte
}

// integrityOf is a packed file's integrity as npm writes it in a lock: sha512- and the base64 of its sha512.
func integrityOf(b []byte) string {
	sum := sha512.Sum512(b)
	return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
}

func digestOf(b []byte) string {
	sum := sha256.Sum256(b)
	return "sha256:" + hex.EncodeToString(sum[:])
}

// packageJSON is the part of a package.json the package steps read.
type packageJSON struct {
	Name    string            `json:"name"`
	Version string            `json:"version"`
	Scripts map[string]string `json:"scripts"`
}

func readPackageJSON(dir string) (packageJSON, error) {
	var p packageJSON
	raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
	if err != nil {
		return p, fmt.Errorf("the package has no package.json: %w", err)
	}
	if err := json.Unmarshal(raw, &p); err != nil {
		return p, fmt.Errorf("the package.json is not JSON: %w", err)
	}
	if p.Name == "" || p.Version == "" {
		return p, fmt.Errorf("the package.json names no name or no version")
	}
	return p, nil
}

// installDev installs a package's own dependencies, its devDependencies included, from its lock (the fetch: network,
// no script, no bin link), and writes a shim for each program they carry into <root>/bin-<name>, files and not links,
// so its build script finds them by name.
func (m *Module) installDev(ctx context.Context, root, dir, rel string) error {
	if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
		return fmt.Errorf("the package in %s has no package-lock.json: commit one", rel)
	}
	npmrc, err := m.npmrcIn(root)
	if err != nil {
		return err
	}
	defer os.Remove(npmrc)
	env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
	r := m.Run(ctx, env, "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc)
	if why := ranFailed(r); why != "" {
		return fmt.Errorf("installing the dependencies of the package in %s: %s", rel, why)
	}
	return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel))
}

// npmrcIn writes the npm configuration for the mesh's registry, with its credential, for one command of this process;
// the caller removes it. It lies in this process's temporary directory, never in a workspace: a run step is given the
// workspace and a /tmp of its own, so no step's mounts hold the credential (ADR 0498 §5, ADR 0500 §4).
func (m *Module) npmrcIn(root string) (string, error) {
	if m.Toolchain == nil || m.Toolchain.Registry == nil {
		return "", fmt.Errorf("the module has no package registry")
	}
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return "", err
	}
	f, err := os.CreateTemp("", ".npmrc-")
	if err != nil {
		return "", err
	}
	defer f.Close()
	if _, err := f.WriteString(reg.Npmrc()); err != nil {
		os.Remove(f.Name())
		return "", err
	}
	return f.Name(), nil
}

// shimDir is where the shims of the package in rel are written.
func shimDir(root, rel string) string {
	return filepath.Join(root, "bin-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
}

// writeShims writes, for every program a top-level package of nm names in its package.json `bin`, a shell script that
// runs it by its real path.
func writeShims(nm, into string) error {
	if err := os.MkdirAll(into, 0o755); err != nil {
		return err
	}
	var dirs []string
	entries, _ := os.ReadDir(nm)
	for _, e := range entries {
		if !e.IsDir() || strings.HasPrefix(e.Name(), ".") {
			continue
		}
		if strings.HasPrefix(e.Name(), "@") {
			scoped, _ := os.ReadDir(filepath.Join(nm, e.Name()))
			for _, s := range scoped {
				if s.IsDir() {
					dirs = append(dirs, filepath.Join(nm, e.Name(), s.Name()))
				}
			}
			continue
		}
		dirs = append(dirs, filepath.Join(nm, e.Name()))
	}
	sort.Strings(dirs)
	for _, d := range dirs {
		raw, err := os.ReadFile(filepath.Join(d, "package.json"))
		if err != nil {
			continue
		}
		var p struct {
			Name string          `json:"name"`
			Bin  json.RawMessage `json:"bin"`
		}
		if json.Unmarshal(raw, &p) != nil || len(p.Bin) == 0 {
			continue
		}
		bins := map[string]string{}
		var one string
		if json.Unmarshal(p.Bin, &one) == nil {
			bins[path.Base(p.Name)] = one
		} else if json.Unmarshal(p.Bin, &bins) != nil {
			continue
		}
		for name, target := range bins {
			if name == "" || strings.ContainsAny(name, "/\\") {
				continue
			}
			body := "#!/bin/sh\nexec node '" + filepath.Join(d, filepath.FromSlash(target)) + "' \"$@\"\n"
			p := filepath.Join(into, name)
			if err := os.WriteFile(p, []byte(body), 0o755); err != nil {
				return err
			}
			if err := os.Chmod(p, 0o755); err != nil {
				return err
			}
		}
	}
	return nil
}

// buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both
// shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never
// stays on the node.
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool) (packed, error) {
	p, err := readPackageJSON(dir)
	if err != nil {
		return packed{}, err
	}
	env := []string{"PATH=" + shimDir(root, rel) + ":" + pathOrDefault(), "npm_config_cache=" + m.npmCache()}
	if p.Scripts["build"] != "" {
		if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "run", "build")); why != "" {
			return packed{}, fmt.Errorf("the build of the package %s: %s", p.Name, why)
		}
	}
	out, err := os.MkdirTemp(root, "packed-")
	if err != nil {
		return packed{}, err
	}
	defer os.RemoveAll(out)
	if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "pack", "--ignore-scripts", "--pack-destination", out)); why != "" {
		return packed{}, fmt.Errorf("packing the package %s: %s", p.Name, why)
	}
	files, _ := filepath.Glob(filepath.Join(out, "*.tgz"))
	if len(files) != 1 {
		return packed{}, fmt.Errorf("npm pack of %s left %d packed files, want one", p.Name, len(files))
	}
	tgz, err := os.ReadFile(files[0])
	if err != nil {
		return packed{}, err
	}
	return packed{Name: p.Name, Version: p.Version, Integrity: integrityOf(tgz), Digest: digestOf(tgz), tgz: tgz}, nil
}

// groupPackage is the fetch's `package` argument: the group's package member (ADR 0500 §5), which the fetch packs and
// serves to its TypeScript members in place of the registry's version.
type groupPackage struct{ Repository, Commit, Ref string }

func groupPackageArg(v any) (*groupPackage, error) {
	if v == nil {
		return nil, nil
	}
	o, ok := v.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("package is {repository, commit, ref}")
	}
	g := &groupPackage{}
	g.Repository, _ = o["repository"].(string)
	g.Commit, _ = o["commit"].(string)
	g.Ref, _ = o["ref"].(string)
	if !repositoryOK.MatchString(g.Repository) || strings.Contains(g.Repository, "..") {
		return nil, fmt.Errorf("package: %q is not a repository, <owner>/<name>", g.Repository)
	}
	if !commitOK.MatchString(g.Commit) {
		return nil, fmt.Errorf("package: %q is not a full commit (40 hex digits)", g.Commit)
	}
	if g.Ref != "" && !strings.HasPrefix(g.Ref, "refs/pull/") {
		return nil, fmt.Errorf("package: %q is not a pull request's head", g.Ref)
	}
	return g, nil
}

// packGroup fetches the group's package member into the workspace, installs its dependencies, builds and packs it,
// publishing nothing (ADR 0500 §4).
func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner, g *groupPackage) (packed, error) {
	dir := filepath.Join(root, "package", "tree")
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return packed{}, err
	}
	if err := fetchCommit(ctx, run, dir, m.remote(g.Repository), g.Commit, g.Ref, m); err != nil {
		return packed{}, fmt.Errorf("the group's package %s: %w", g.Repository, err)
	}
	if err := m.installDev(ctx, root, dir, "package"); err != nil {
		return packed{}, err
	}
	return m.buildAndPack(ctx, root, dir, "package", false)
}

// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is
// false when the lock names none (the toolchain's SDK is then the module's, as before).
func sdkLocked(dir string) (version, integrity string, ok bool, err error) {
	raw, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
	if os.IsNotExist(err) {
		return "", "", false, nil
	}
	if err != nil {
		return "", "", false, err
	}
	var lock struct {
		Packages map[string]struct {
			Version   string `json:"version"`
			Integrity string `json:"integrity"`
		} `json:"packages"`
	}
	if err := json.Unmarshal(raw, &lock); err != nil {
		return "", "", false, fmt.Errorf("the package-lock.json is not JSON: %w", err)
	}
	p, ok := lock.Packages["node_modules/"+sdkPackage]
	if !ok || p.Version == "" {
		return "", "", false, nil
	}
	return p.Version, p.Integrity, true, nil
}

// placeSDK places in the module's node_modules the SDK version its lock names (the lead's decision L10): the group's
// packed package when it is that version (its integrity must be the lock's), else the registry's; a version neither
// holds is a wait (`waits for:`) when no group package was given, and a failure naming both when one was.
func (m *Module) placeSDK(ctx context.Context, dir, rel string, group *packed) error {
	version, integrity, ok, err := sdkLocked(dir)
	if err != nil || !ok {
		return err
	}
	want := sdkPackage + "@" + version
	var tgz []byte
	switch {
	case group != nil && group.Name == sdkPackage && group.Version == version:
		if integrity != "" && integrity != group.Integrity {
			return fmt.Errorf("the lock of %s names %s with the integrity %s, and the group's package packs to %s", rel, want, integrity, group.Integrity)
		}
		tgz = group.tgz
	default:
		got, held, err := m.registryPackage(ctx, sdkPackage, version)
		if err != nil {
			return err
		}
		if !held {
			if group != nil {
				return fmt.Errorf("the lock of %s names %s, which neither the registry nor the group (%s@%s) holds", rel, want, group.Name, group.Version)
			}
			return fmt.Errorf("%s%s", delivery.WaitsFor, want)
		}
		if integrity != "" && integrityOf(got) != integrity {
			return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity)
		}
		tgz = got
	}
	into := filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage))
	if err := os.RemoveAll(into); err != nil {
		return err
	}
	return unpackPackage(tgz, into)
}

// httpClient is the client of the registry's reads and publishes: HTTP when a test set it.
func (m *Module) httpClient() *http.Client {
	if m.HTTP != nil {
		return m.HTTP
	}
	return &http.Client{Timeout: 5 * time.Minute}
}

// packument is the part of the registry's document of a package the steps read: each version's packed file.
type packument struct {
	Versions map[string]struct {
		Dist struct {
			Tarball   string `json:"tarball"`
			Integrity string `json:"integrity"`
		} `json:"dist"`
	} `json:"versions"`
}

// registryVersion reads what the registry holds of name@version: its packed file's address and integrity, and whether
// it holds it at all.
func (m *Module) registryVersion(ctx context.Context, name, version string) (tarball, integrity string, held bool, err error) {
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return "", "", false, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reg.URL+url.PathEscape(name), nil)
	if err != nil {
		return "", "", false, err
	}
	reg.authorize(req)
	resp, err := m.httpClient().Do(req)
	if err != nil {
		return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		return "", "", false, nil
	}
	if resp.StatusCode != http.StatusOK {
		said, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return "", "", false, fmt.Errorf("the registry answered %s for %s: %s", resp.Status, name, strings.TrimSpace(string(said)))
	}
	var p packument
	if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<20)).Decode(&p); err != nil {
		return "", "", false, fmt.Errorf("the registry's document of %s: %w", name, err)
	}
	v, ok := p.Versions[version]
	if !ok {
		return "", "", false, nil
	}
	return v.Dist.Tarball, v.Dist.Integrity, true, nil
}

// registryPackage downloads name@version's packed file from the registry; held is false when the registry has none.
func (m *Module) registryPackage(ctx context.Context, name, version string) ([]byte, bool, error) {
	tarball, _, held, err := m.registryVersion(ctx, name, version)
	if err != nil || !held {
		return nil, held, err
	}
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return nil, false, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil)
	if err != nil {
		return nil, false, err
	}
	reg.authorize(req)
	resp, err := m.httpClient().Do(req)
	if err != nil {
		return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, false, fmt.Errorf("the registry answered %s for %s@%s's packed file", resp.Status, name, version)
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, maxPackage+1))
	if err != nil {
		return nil, false, err
	}
	if len(body) > maxPackage {
		return nil, false, fmt.Errorf("%s@%s is larger than %d bytes", name, version, maxPackage)
	}
	return body, true, nil
}

// unpackPackage unpacks a packed npm package (a gzipped tar whose entries lie under package/) into dir: files and
// directories only, nothing outside.
func unpackPackage(tgz []byte, dir string) error {
	zr, err := gzip.NewReader(bytes.NewReader(tgz))
	if err != nil {
		return fmt.Errorf("the packed package is not gzip: %w", err)
	}
	tr := tar.NewReader(zr)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		name := path.Clean(strings.TrimPrefix(h.Name, "./"))
		rel, ok := strings.CutPrefix(name, "package/")
		if !ok || rel == "" || path.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, "../") {
			if name == "package" {
				continue
			}
			return fmt.Errorf("the packed package holds %s, outside package/", h.Name)
		}
		target := filepath.Join(dir, filepath.FromSlash(rel))
		switch h.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, 0o755); err != nil {
				return err
			}
		case tar.TypeReg:
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return err
			}
			body, err := io.ReadAll(io.LimitReader(tr, maxPackage))
			if err != nil {
				return err
			}
			if err := os.WriteFile(target, body, os.FileMode(h.Mode).Perm()|0o600); err != nil {
				return err
			}
		default:
			return fmt.Errorf("the packed package holds %s, neither a file nor a directory", h.Name)
		}
	}
}

// fetchCommit fetches one commit into dir as the fetch step fetches the module's own (buildstep.FetchCommit).
func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, m *Module) error {
	return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...)
}

// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. Only a trunk build
// (publish) publishes it, from this module's own process with the registry's write credential, which no run step is
// given: a version the registry lacks is published; the same packed file already there is done without a publish;
// another packed file under that version fails `version taken:`. The reference is <name>@<version>, answered only when
// the registry holds this build's file; a candidate answers its digest alone.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) {
	rel := module
	if rel == "" {
		rel = "."
	}
	p, err := m.buildAndPack(ctx, root, dir, rel, publish)
	if err != nil {
		return nil, err
	}
	res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)}
	if publish {
		if res.Reference, err = m.publish(ctx, root, p); err != nil {
			return nil, err
		}
	}
	return res, nil
}

// publish puts a packed package into the registry unless it holds that version, and answers <name>@<version>.
func (m *Module) publish(ctx context.Context, root string, p packed) (string, error) {
	ref := p.Name + "@" + p.Version
	_, integrity, held, err := m.registryVersion(ctx, p.Name, p.Version)
	if err != nil {
		return "", err
	}
	if held {
		if integrity == p.Integrity {
			return ref, nil // this very file, published before: done without a publish
		}
		return "", fmt.Errorf("%s%s: the registry holds it as %s, and this build packs %s", delivery.VersionTaken, ref, integrity, p.Integrity)
	}
	file, err := os.CreateTemp(root, "publish-*.tgz")
	if err != nil {
		return "", err
	}
	defer os.Remove(file.Name()) // the packed file leaves the node once its publish answered
	if _, err := file.Write(p.tgz); err != nil {
		file.Close()
		return "", err
	}
	if err := file.Close(); err != nil {
		return "", err
	}
	npmrc, err := m.npmrcIn(root)
	if err != nil {
		return "", err
	}
	defer os.Remove(npmrc)
	env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
	r := m.Run(ctx, env, "npm", "publish", file.Name(), "--ignore-scripts", "--userconfig", npmrc)
	if why := ranFailed(r); why != "" {
		return "", fmt.Errorf("publishing %s: %s", ref, why)
	}
	return ref, nil
}