mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/fetch.go

package main

// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
//
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.

import (
	"context"
	"encoding/json"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
const ForgeAlias = "mesh-forge"

// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
const sdkPackage = "@novox/mesh-sdk"

var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)

var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)

// FetchResult is what a passed fetch step says.
type FetchResult struct {
	Workspace string `json:"workspace"`
	Commit    string `json:"commit"`
	// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
	Modules   []string `json:"modules"`
	Installed []string `json:"installed"`
	// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
	// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
	Refused []Refusal `json:"refused"`
	// The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk
	// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
	// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
	// left at the trunk commit with nothing installed.
	Base      string                          `json:"base,omitempty"`
	Merge     string                          `json:"merge,omitempty"`
	Conflicts []string                        `json:"conflicts,omitempty"`
	Changed   []string                        `json:"changed,omitempty"`
	Trees     map[string]buildstep.ModuleTree `json:"trees,omitempty"`
	// Group is the group's package the fetch packed and served to the TypeScript members (the call's `package`):
	// "<name>@<version> <integrity>".
	Group string `json:"group,omitempty"`
}

// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
const narrowedMark = "narrowed"

// narrowed says whether the fetch narrowed the workspace whose tree this is.
func narrowed(tree string) bool {
	_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
	return err == nil
}

// Refusal is a module a fetch did not install the packages of.
type Refusal struct {
	Module string `json:"module"`
	Reason string `json:"reason"`
}

func (m *Module) fetchTool() StepTool {
	return StepTool{
		Name: "build_node_fetch",
		Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
			"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
			"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
		Input: map[string]any{
			"type": "object",
			"properties": map[string]any{
				"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
				"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
				"ref":        map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
				"dir":        buildstep.FetchRequestProperties()["dir"],
				"paths":      buildstep.FetchRequestProperties()["paths"],
				"package": map[string]any{"type": "object", "description": "optional: the group's package member (ADR 0500 §5), packed here, publishing nothing, and served to each TypeScript member whose lock names its version, in place of the registry's",
					"properties": map[string]any{"repository": map[string]any{"type": "string"}, "commit": map[string]any{"type": "string"}, "ref": map[string]any{"type": "string"}},
					"required":   []string{"repository", "commit"}},
				"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
			},
			"required": []string{"repository", "commit"},
		},
		Long: true,
		Run:  m.fetch,
	}
}

// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
// since nobody learns its id to release it (issue 1839).
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
	// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
	// named paths, the workspace is narrowed to them (buildstep.Narrow).
	req, err := buildstep.ParseFetchRequest(args)
	if err != nil {
		return nil, err
	}
	group, err := groupPackageArg(args["package"])
	if err != nil {
		return nil, err
	}
	m.CleanWorkspaces(m.now())
	id := buildstep.NewID()
	done := m.spaces().Hold(id)
	defer func() {
		done()
		if err != nil {
			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
				m.Say("the failed fetch's workspace %s: %v", id, rerr)
			}
		}
	}()
	return m.fetchInto(ctx, req, group, id)
}

func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) {
	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
	var err error
	root := filepath.Join(m.Work, id)
	tree := filepath.Join(root, "tree")
	if err := os.MkdirAll(tree, 0o700); err != nil {
		return nil, err
	}
	// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
	gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
		return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
	}
	remote := m.remote(repository)
	if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil {
		return nil, err
	}
	res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
	if base != "" {
		if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil {
			return nil, err
		}
		if res.Merge == "" {
			return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
		}
	}
	if len(req.Paths) > 0 {
		if err := buildstep.Narrow(tree, req); err != nil {
			return nil, err
		}
		// Said beside the tree, so the listing of the tree's modules leaves out those narrowed away.
		if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
			return nil, err
		}
	}
	// The group's package, packed here and served to its members in place of the registry's version (ADR 0500 §5).
	var served *packed
	if group != nil {
		p, err := m.packGroup(ctx, root, gitRun, group)
		if err != nil {
			return nil, err
		}
		served, res.Group = &p, p.Name+"@"+p.Version+" "+p.Integrity
	}
	dirs, packages, err := m.typeScriptModules(ctx, tree)
	if err != nil {
		return nil, err
	}
	for _, d := range dirs {
		res.Modules = append(res.Modules, d)
		dir := filepath.Join(tree, filepath.FromSlash(d))
		if packages[d] {
			// A package is built from its own lock's dependencies, its devDependencies included (ADR 0498 §5).
			if err := m.installDev(ctx, root, dir, d); err != nil {
				return nil, err
			}
			res.Installed = append(res.Installed, d)
			continue
		}
		did, refused, err := m.installOwn(ctx, root, dir, d)
		if err != nil {
			return nil, err
		}
		if refused == "" {
			// The SDK the module's own lock names, from the group or the registry (the lead's decision L10).
			if err := m.placeSDK(ctx, dir, d, served); err != nil {
				return nil, err
			}
		}
		if refused != "" {
			res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
		}
		if did {
			res.Installed = append(res.Installed, d)
		}
	}
	return res, nil
}

// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
// it, in order.
func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) {
	r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
	if r.Status != 0 || r.Err != "" {
		return nil, nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
	}
	packages = map[string]bool{}
	for _, l := range strings.Split(r.Stdout, "\n") {
		if l = strings.TrimSpace(l); l == "" {
			continue
		}
		raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
		if os.IsNotExist(err) && narrowed(tree) {
			continue // outside a narrowed workspace's paths (buildstep.Narrow): no module of this build
		}
		if err != nil {
			return nil, nil, err
		}
		var mj struct {
			Build struct {
				Artifacts []artifact `json:"artifacts"`
			} `json:"build"`
		}
		if json.Unmarshal(raw, &mj) != nil {
			continue // a module.json the build step will refuse by name, not a fetch's business
		}
		for _, a := range mj.Build.Artifacts {
			if a.Language == "typescript" {
				d := filepath.ToSlash(filepath.Dir(l))
				dirs = append(dirs, d)
				packages[d] = a.Kind == "package"
				break
			}
		}
	}
	sort.Strings(dirs)
	return dirs, packages, nil
}

// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
	raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
	if os.IsNotExist(err) {
		return nil, nil, nil
	}
	if err != nil {
		return nil, nil, err
	}
	if err := json.Unmarshal(raw, &pkg); err != nil {
		return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
	}
	deps, _ := pkg["dependencies"].(map[string]any)
	for name := range deps {
		if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	return names, pkg, nil
}

// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no
// package-lock.json. Empty when it names none or has its lock.
func lockProblem(dir, rel string) (string, error) {
	names, _, err := ownDependencies(dir)
	if err != nil || len(names) == 0 {
		return "", err
	}
	if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
		return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil
	}
	return "", nil
}

// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module
// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and
// the build of that bundle fails naming it.
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) {
	names, pkg, err := ownDependencies(dir)
	if err != nil || len(names) == 0 {
		return false, "", err
	}
	if why, err := lockProblem(dir, rel); why != "" || err != nil {
		return false, why, err
	}
	lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
	if err != nil {
		return false, "", err
	}
	// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
	scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
	if err := os.MkdirAll(scratch, 0o700); err != nil {
		return false, "", err
	}
	for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
		if d, ok := pkg[k].(map[string]any); ok {
			delete(d, sdkPackage)
		}
	}
	delete(pkg, "devDependencies")
	delete(pkg, "scripts")
	raw, _ := json.Marshal(pkg)
	if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
		return false, "", err
	}
	if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
		return false, "", err
	}
	// The registry is given through a config file of its own, never on the command line, outside the workspace, and
	// removed after (npmrcIn).
	npmrc, err := m.npmrcIn(root)
	if err != nil {
		return false, "", fmt.Errorf("installing the packages of %s: %w", rel, err)
	}
	env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
	r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
		"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
	os.Remove(npmrc)
	if why := ranFailed(r); why != "" {
		return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
	}
	nm := filepath.Join(scratch, "node_modules")
	if err := removeSDKCopies(nm); err != nil {
		return false, "", err
	}
	if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
		return false, "", err
	}
	if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
		return false, "", err
	}
	return true, "", os.RemoveAll(scratch)
}

// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
func removeSDKCopies(nm string) error {
	var found []string
	err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
			found = append(found, p)
			return fs.SkipDir
		}
		return nil
	})
	if err != nil {
		return err
	}
	for _, p := range found {
		if err := os.RemoveAll(p); err != nil {
			return err
		}
	}
	return nil
}

// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
// net for a caller that never released one (issue 1839, ADR 0509).
func (m *Module) CleanWorkspaces(now time.Time) {
	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
}