mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/package.go

package main

// A TypeScript package (novox/hq ADR 0498 §5, ADR 0500 §4-§6; issue 1587, the lead's decision L10): built from its own
// lock's devDependencies (the toolchain's SDK is never used to build the SDK itself), packed with npm pack, and, for a
// group, served to its TypeScript members' fetch in place of the registry's version. A member is built against the SDK
// version its own lock names: the fetch places that version in the member's node_modules, from the group's packed
// package when the group carries that version, else from the registry; a version neither holds waits for its
// publication (`waits for:`). The install of a package's devDependencies is the fetch's (it holds the network and runs
// no script); its build and pack run shut off like every run step.

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"crypto/sha512"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
	builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// maxPackage is the largest packed package read: far above the SDK's.
const maxPackage = 64 << 20

// packed is a package built and packed: its name and version from its package.json, the packed file's bytes, its npm
// integrity (sha512, as a lock names it) and its digest (sha256, as a done key names it).
type packed struct {
	Name, Version, Integrity, Digest string
	tgz                              []byte
}

// integrityOf is a packed file's integrity as npm writes it in a lock: sha512- and the base64 of its sha512.
func integrityOf(b []byte) string {
	sum := sha512.Sum512(b)
	return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
}

func digestOf(b []byte) string {
	sum := sha256.Sum256(b)
	return "sha256:" + hex.EncodeToString(sum[:])
}

// packageJSON is the part of a package.json the package steps read.
type packageJSON struct {
	Name    string            `json:"name"`
	Version string            `json:"version"`
	Scripts map[string]string `json:"scripts"`
}

func readPackageJSON(dir string) (packageJSON, error) {
	var p packageJSON
	raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
	if err != nil {
		return p, fmt.Errorf("the package has no package.json: %w", err)
	}
	if err := json.Unmarshal(raw, &p); err != nil {
		return p, fmt.Errorf("the package.json is not JSON: %w", err)
	}
	if p.Name == "" || p.Version == "" {
		return p, fmt.Errorf("the package.json names no name or no version")
	}
	return p, nil
}

// installDev installs a package's own dependencies, its devDependencies included, from its lock (the fetch: network,
// no script, no bin link), and writes a shim for each program they carry into <root>/bin-<name>, files and not links,
// so its build script finds them by name.
func (m *Module) installDev(ctx context.Context, root, dir, rel string) error {
	if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
		return fmt.Errorf("the package in %s has no package-lock.json: commit one", rel)
	}
	npmrc, err := m.npmrcIn(root)
	if err != nil {
		return err
	}
	defer os.Remove(npmrc)
	env := append(m.baseEnv(), "npm_config_cache="+m.npmCache())
	r := m.Run(ctx, env, "npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc)
	if why := ranFailed(r); why != "" {
		return fmt.Errorf("installing the dependencies of the package in %s: %s", rel, why)
	}
	return writeShims(filepath.Join(dir, "node_modules"), shimDir(root, rel))
}

// npmrcIn writes the npm configuration for the mesh's registry, read anonymously, for one command of this process; the
// caller removes it. It lies in this process's temporary directory, never in a workspace (ADR 0498 §5, ADR 0500 §4).
func (m *Module) npmrcIn(root string) (string, error) {
	if m.Toolchain == nil || m.Toolchain.Registry == nil {
		return "", fmt.Errorf("the module has no package registry")
	}
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return "", err
	}
	f, err := os.CreateTemp("", ".npmrc-")
	if err != nil {
		return "", err
	}
	defer f.Close()
	if _, err := f.WriteString(reg.Npmrc()); err != nil {
		os.Remove(f.Name())
		return "", err
	}
	return f.Name(), nil
}

// shimDir is where the shims of the package in rel are written.
func shimDir(root, rel string) string {
	return filepath.Join(root, "bin-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
}

// writeShims writes, for every program a top-level package of nm names in its package.json `bin`, a shell script that
// runs it by its real path.
func writeShims(nm, into string) error {
	if err := os.MkdirAll(into, 0o755); err != nil {
		return err
	}
	var dirs []string
	entries, _ := os.ReadDir(nm)
	for _, e := range entries {
		if !e.IsDir() || strings.HasPrefix(e.Name(), ".") {
			continue
		}
		if strings.HasPrefix(e.Name(), "@") {
			scoped, _ := os.ReadDir(filepath.Join(nm, e.Name()))
			for _, s := range scoped {
				if s.IsDir() {
					dirs = append(dirs, filepath.Join(nm, e.Name(), s.Name()))
				}
			}
			continue
		}
		dirs = append(dirs, filepath.Join(nm, e.Name()))
	}
	sort.Strings(dirs)
	for _, d := range dirs {
		raw, err := os.ReadFile(filepath.Join(d, "package.json"))
		if err != nil {
			continue
		}
		var p struct {
			Name string          `json:"name"`
			Bin  json.RawMessage `json:"bin"`
		}
		if json.Unmarshal(raw, &p) != nil || len(p.Bin) == 0 {
			continue
		}
		bins := map[string]string{}
		var one string
		if json.Unmarshal(p.Bin, &one) == nil {
			bins[path.Base(p.Name)] = one
		} else if json.Unmarshal(p.Bin, &bins) != nil {
			continue
		}
		for name, target := range bins {
			if name == "" || strings.ContainsAny(name, "/\\") {
				continue
			}
			body := "#!/bin/sh\nexec node '" + filepath.Join(d, filepath.FromSlash(target)) + "' \"$@\"\n"
			p := filepath.Join(into, name)
			if err := os.WriteFile(p, []byte(body), 0o755); err != nil {
				return err
			}
			if err := os.Chmod(p, 0o755); err != nil {
				return err
			}
		}
	}
	return nil
}

// buildAndPack builds the package in dir with its own build script, if it has one, and packs it with npm pack, both
// shut off (write: a trunk build's, which alone writes the shared cache). The packed file is read and removed: it never
// stays on the node.
func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write bool, version string) (packed, error) {
	if version != "" {
		restore, err := setPackageVersion(dir, version)
		if err != nil {
			return packed{}, err
		}
		defer restore()
	}
	p, err := readPackageJSON(dir)
	if err != nil {
		return packed{}, err
	}
	env := []string{"PATH=" + shimDir(root, rel) + ":" + pathOrDefault(), "npm_config_cache=" + m.npmCache()}
	if p.Scripts["build"] != "" {
		if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "run", "build")); why != "" {
			return packed{}, fmt.Errorf("the build of the package %s: %s", p.Name, why)
		}
	}
	out, err := os.MkdirTemp(root, "packed-")
	if err != nil {
		return packed{}, err
	}
	defer os.RemoveAll(out)
	if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "pack", "--ignore-scripts", "--pack-destination", out)); why != "" {
		return packed{}, fmt.Errorf("packing the package %s: %s", p.Name, why)
	}
	files, _ := filepath.Glob(filepath.Join(out, "*.tgz"))
	if len(files) != 1 {
		return packed{}, fmt.Errorf("npm pack of %s left %d packed files, want one", p.Name, len(files))
	}
	tgz, err := os.ReadFile(files[0])
	if err != nil {
		return packed{}, err
	}
	return packed{Name: p.Name, Version: p.Version, Integrity: integrityOf(tgz), Digest: digestOf(tgz), tgz: tgz}, nil
}

// groupPackage is the fetch's `package` argument: the group's package member (ADR 0500 §5), which the fetch packs and
// serves to its TypeScript members in place of the registry's version, or, by Name and Version, the group's tagged
// version of the SDK, which the fetch places from the registry in each TypeScript member, whatever version its lock
// names (novox/hq ADR 0607 §3, §5; design 51 *Tagged versions*): nothing is packed then. The packed member is the old
// form, kept because the running checker hands it until the checker that hands the tagged version is delivered; its
// removal is a subtask of the milestone after that delivery (issue 1966).
type groupPackage struct{ Repository, Commit, Ref, Name, Version string }

// tagged says whether the argument names a tagged version to take from the registry, not a member to pack.
func (g *groupPackage) tagged() bool { return g != nil && g.Version != "" }

func groupPackageArg(v any) (*groupPackage, error) {
	if v == nil {
		return nil, nil
	}
	o, ok := v.(map[string]any)
	if !ok {
		return nil, fmt.Errorf("package is {repository, commit, ref} or {name, version}")
	}
	g := &groupPackage{}
	g.Name, _ = o["name"].(string)
	g.Version, _ = o["version"].(string)
	if g.Name != "" || g.Version != "" {
		if g.Name != sdkPackage {
			return nil, fmt.Errorf("package: %q is not %s, the one package the fetch places in a member", g.Name, sdkPackage)
		}
		if !npmVersionOK.MatchString(g.Version) {
			return nil, fmt.Errorf("package: %q is not a version (0.1.15-feat-x.2)", g.Version)
		}
		return g, nil
	}
	g.Repository, _ = o["repository"].(string)
	g.Commit, _ = o["commit"].(string)
	g.Ref, _ = o["ref"].(string)
	if !repositoryOK.MatchString(g.Repository) || strings.Contains(g.Repository, "..") {
		return nil, fmt.Errorf("package: %q is not a repository, <owner>/<name>", g.Repository)
	}
	if !commitOK.MatchString(g.Commit) {
		return nil, fmt.Errorf("package: %q is not a full commit (40 hex digits)", g.Commit)
	}
	if g.Ref != "" && !strings.HasPrefix(g.Ref, "refs/pull/") {
		return nil, fmt.Errorf("package: %q is not a pull request's head", g.Ref)
	}
	return g, nil
}

// npmVersionOK is a semantic version as npm names one, with an optional pre-release (a tagged version's).
var npmVersionOK = regexp.MustCompile(`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)

// packGroup fetches the group's package member into the workspace, installs its dependencies, builds and packs it,
// publishing nothing (ADR 0500 §4).
func (m *Module) packGroup(ctx context.Context, root string, run forgekey.Runner, g *groupPackage) (packed, error) {
	dir := filepath.Join(root, "package", "tree")
	if err := os.MkdirAll(dir, 0o700); err != nil {
		return packed{}, err
	}
	if err := fetchCommit(ctx, run, dir, m.remote(g.Repository), g.Commit, g.Ref, m); err != nil {
		return packed{}, fmt.Errorf("the group's package %s: %w", g.Repository, err)
	}
	if err := m.installDev(ctx, root, dir, "package"); err != nil {
		return packed{}, err
	}
	return m.buildAndPack(ctx, root, dir, "package", false, "")
}

// sdkLocked is what a module's lock names of the SDK: its version, its integrity and where it was resolved from; ok is
// false when the lock names none (the toolchain's SDK is then the module's, as before).
func sdkLocked(dir string) (version, integrity string, ok bool, err error) {
	raw, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
	if os.IsNotExist(err) {
		return "", "", false, nil
	}
	if err != nil {
		return "", "", false, err
	}
	var lock struct {
		Packages map[string]struct {
			Version   string `json:"version"`
			Integrity string `json:"integrity"`
		} `json:"packages"`
	}
	if err := json.Unmarshal(raw, &lock); err != nil {
		return "", "", false, fmt.Errorf("the package-lock.json is not JSON: %w", err)
	}
	p, ok := lock.Packages["node_modules/"+sdkPackage]
	if !ok || p.Version == "" {
		return "", "", false, nil
	}
	return p.Version, p.Integrity, true, nil
}

// placeSDK places in the module's node_modules the SDK version its lock names (the lead's decision L10): the group's
// packed package when it is that version (its integrity must be the lock's), else the registry's; a version neither
// holds is a wait (`waits for:`) when no group package was given, and a failure naming both when one was. Given the
// group's tagged version (tagged, ADR 0607 §3), that version is placed from the registry whatever the lock names, its
// integrity the lock's when the lock names it, else the registry's own; one the registry lacks fails naming it.
func (m *Module) placeSDK(ctx context.Context, dir, rel string, group *packed, tagged *groupPackage) error {
	version, integrity, ok, err := sdkLocked(dir)
	if err != nil || !ok {
		return err
	}
	var tgz []byte
	switch {
	case tagged.tagged():
		if version != tagged.Version {
			if m.Say != nil {
				m.Say("%s: its lock names %s@%s; the group's tagged version %s is placed (ADR 0607 §3)", rel, sdkPackage, version, tagged.Version)
			}
			version, integrity = tagged.Version, ""
		}
		want := sdkPackage + "@" + version
		got, registered, held, err := m.registryPackageChecked(ctx, sdkPackage, version)
		if err != nil {
			return err
		}
		if !held {
			return fmt.Errorf("the registry holds no %s, the group's tagged version, so %s cannot be built against it", want, rel)
		}
		if integrity == "" {
			integrity = registered
		}
		if integrity != "" && integrityOf(got) != integrity {
			return fmt.Errorf("the registry's %s is %s, and %s names %s", want, integrityOf(got), rel, integrity)
		}
		tgz = got
	case group != nil && group.Name == sdkPackage && group.Version == version:
		if integrity != "" && integrity != group.Integrity {
			return fmt.Errorf("the lock of %s names %s with the integrity %s, and the group's package packs to %s", rel, sdkPackage+"@"+version, integrity, group.Integrity)
		}
		tgz = group.tgz
	default:
		want := sdkPackage + "@" + version
		got, held, err := m.registryPackage(ctx, sdkPackage, version)
		if err != nil {
			return err
		}
		if !held {
			if group != nil {
				return fmt.Errorf("the lock of %s names %s, which neither the registry nor the group (%s@%s) holds", rel, want, group.Name, group.Version)
			}
			return fmt.Errorf("%s%s", builds.WaitsFor, want)
		}
		if integrity != "" && integrityOf(got) != integrity {
			return fmt.Errorf("the registry's %s is %s, and the lock of %s names %s", want, integrityOf(got), rel, integrity)
		}
		tgz = got
	}
	into := filepath.Join(dir, "node_modules", filepath.FromSlash(sdkPackage))
	if err := os.RemoveAll(into); err != nil {
		return err
	}
	return unpackPackage(tgz, into)
}

// registryPackageChecked is registryPackage with the integrity the registry's document names of name@version.
func (m *Module) registryPackageChecked(ctx context.Context, name, version string) (tgz []byte, integrity string, held bool, err error) {
	_, integrity, held, err = m.registryVersion(ctx, name, version)
	if err != nil || !held {
		return nil, "", held, err
	}
	tgz, held, err = m.registryPackage(ctx, name, version)
	return tgz, integrity, held, err
}

// httpClient is the client of the registry's reads and publishes: HTTP when a test set it.
func (m *Module) httpClient() *http.Client {
	if m.HTTP != nil {
		return m.HTTP
	}
	return &http.Client{Timeout: 5 * time.Minute}
}

// packument is the part of the registry's document of a package the steps read: each version's packed file.
type packument struct {
	Versions map[string]struct {
		Dist struct {
			Tarball   string `json:"tarball"`
			Integrity string `json:"integrity"`
		} `json:"dist"`
	} `json:"versions"`
}

// registryVersion reads what the registry holds of name@version: its packed file's address and integrity, and whether
// it holds it at all.
func (m *Module) registryVersion(ctx context.Context, name, version string) (tarball, integrity string, held bool, err error) {
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return "", "", false, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reg.URL+url.PathEscape(name), nil)
	if err != nil {
		return "", "", false, err
	}
	resp, err := m.httpClient().Do(req)
	if err != nil {
		return "", "", false, fmt.Errorf("asking the registry for %s: %w", name, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		return "", "", false, nil
	}
	if resp.StatusCode != http.StatusOK {
		said, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return "", "", false, fmt.Errorf("the registry answered %s for %s: %s", resp.Status, name, strings.TrimSpace(string(said)))
	}
	var p packument
	if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<20)).Decode(&p); err != nil {
		return "", "", false, fmt.Errorf("the registry's document of %s: %w", name, err)
	}
	v, ok := p.Versions[version]
	if !ok {
		return "", "", false, nil
	}
	return v.Dist.Tarball, v.Dist.Integrity, true, nil
}

// registryPackage downloads name@version's packed file from the registry; held is false when the registry has none.
func (m *Module) registryPackage(ctx context.Context, name, version string) ([]byte, bool, error) {
	tarball, _, held, err := m.registryVersion(ctx, name, version)
	if err != nil || !held {
		return nil, held, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tarball, nil)
	if err != nil {
		return nil, false, err
	}
	resp, err := m.httpClient().Do(req)
	if err != nil {
		return nil, false, fmt.Errorf("downloading %s@%s: %w", name, version, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, false, fmt.Errorf("the registry answered %s for %s@%s's packed file", resp.Status, name, version)
	}
	body, err := io.ReadAll(io.LimitReader(resp.Body, maxPackage+1))
	if err != nil {
		return nil, false, err
	}
	if len(body) > maxPackage {
		return nil, false, fmt.Errorf("%s@%s is larger than %d bytes", name, version, maxPackage)
	}
	return body, true, nil
}

// unpackPackage unpacks a packed npm package (a gzipped tar whose entries lie under package/) into dir: files and
// directories only, nothing outside.
func unpackPackage(tgz []byte, dir string) error {
	zr, err := gzip.NewReader(bytes.NewReader(tgz))
	if err != nil {
		return fmt.Errorf("the packed package is not gzip: %w", err)
	}
	tr := tar.NewReader(zr)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		name := path.Clean(strings.TrimPrefix(h.Name, "./"))
		rel, ok := strings.CutPrefix(name, "package/")
		if !ok || rel == "" || path.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, "../") {
			if name == "package" {
				continue
			}
			return fmt.Errorf("the packed package holds %s, outside package/", h.Name)
		}
		target := filepath.Join(dir, filepath.FromSlash(rel))
		switch h.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, 0o755); err != nil {
				return err
			}
		case tar.TypeReg:
			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
				return err
			}
			body, err := io.ReadAll(io.LimitReader(tr, maxPackage))
			if err != nil {
				return err
			}
			if err := os.WriteFile(target, body, os.FileMode(h.Mode).Perm()|0o600); err != nil {
				return err
			}
		default:
			return fmt.Errorf("the packed package holds %s, neither a file nor a directory", h.Name)
		}
	}
}

// fetchCommit fetches one commit into dir as the fetch step fetches the module's own (buildstep.FetchCommit).
func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, m *Module) error {
	return buildstep.FetchCommit(ctx, run, dir, remote, commit, ref, m.FetchOptions...)
}

// buildPackage builds and packs a package artifact (ADR 0498 §5): its digest is the packed file's. A trunk build
// (publish) the builder gave a version packs it under that version and publishes it through npm-package-registry
// (design 51 *Publishing*, CAT-BN3), from this module's own process, which holds no write credential: the holder asks
// the builder, answers held for the same file and refuses another under that version. The reference is
// <name>@<version>, answered only when the holder took it; a candidate, or a trunk build not numbered, answers its
// digest alone. dir is the package's directory, at its place in the module's.
func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir, at string, a artifact, publish bool, version, seat string) (any, error) {
	// rel is the package's directory in the workspace, as the fetch installed it (the shims lie under it).
	rel := path.Join(module, at)
	if module == "" {
		rel = path.Join(".", at)
	}
	if !publish {
		version = ""
	}
	if seat == "" {
		seat = buildstep.NpmSeat
	}
	if version != "" && seat != buildstep.NpmSeat {
		return nil, fmt.Errorf("the package %s is published through %s, not %s: this module publishes npm packages", a.Name, buildstep.NpmSeat, seat)
	}
	p, err := m.buildAndPack(ctx, root, dir, rel, publish, version)
	if err != nil {
		return nil, err
	}
	res := BuildResult{Workspace: workspace, Module: module, Artifact: a.Name, Digest: p.Digest, Bytes: len(p.tgz)}
	if version == "" {
		return res, nil
	}
	if err := m.publishOn(ctx, seat, p.Name, "package", p.Version, p.Digest, p.tgz); err != nil {
		return nil, fmt.Errorf("%s%s: %v", builds.NotStored, a.Name, err)
	}
	res.Reference = p.Name + "@" + p.Version
	return res, nil
}

// setPackageVersion writes version as dir's package.json `version` (the builder's, design 51 *The versions*), so npm
// packs the package under it; restore puts the file back as it was.
func setPackageVersion(dir, version string) (restore func(), err error) {
	path := filepath.Join(dir, "package.json")
	raw, err := os.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("the package's package.json: %w", err)
	}
	var doc map[string]json.RawMessage
	if err := json.Unmarshal(raw, &doc); err != nil {
		return nil, fmt.Errorf("the package's package.json does not parse: %w", err)
	}
	doc["version"], _ = json.Marshal(version)
	out, err := json.MarshalIndent(doc, "", "  ")
	if err != nil {
		return nil, err
	}
	if err := os.WriteFile(path, append(out, '\n'), 0o644); err != nil {
		return nil, err
	}
	return func() { _ = os.WriteFile(path, raw, 0o644) }, nil
}