mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/fetch.go

package main

// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
//
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.

import (
	"context"
	"encoding/json"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
const ForgeAlias = "mesh-forge"

// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
const sdkPackage = "@novox/mesh-sdk"

// WorkspaceAge is how long a workspace is kept.
const WorkspaceAge = 24 * time.Hour

var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)

// FetchResult is what a passed fetch step says.
type FetchResult struct {
	Workspace string `json:"workspace"`
	Commit    string `json:"commit"`
	// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
	Modules   []string `json:"modules"`
	Installed []string `json:"installed"`
	// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
	// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
	Refused []Refusal `json:"refused"`
}

// Refusal is a module a fetch did not install the packages of.
type Refusal struct {
	Module string `json:"module"`
	Reason string `json:"reason"`
}

func (m *Module) fetchTool() StepTool {
	return StepTool{
		Name: "build_node_fetch",
		Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
			"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
			"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
		Input: map[string]any{
			"type": "object",
			"properties": map[string]any{
				"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
				"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
				"ref":        map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
			},
			"required": []string{"repository", "commit"},
		},
		Long: true,
		Run:  m.fetch,
	}
}

func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
	repository, _ := args["repository"].(string)
	commit, _ := args["commit"].(string)
	ref, _ := args["ref"].(string)
	if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
		return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
	}
	m.CleanWorkspaces(m.now())
	id := buildstep.NewID()
	root := filepath.Join(m.Work, id)
	tree := filepath.Join(root, "tree")
	if err := os.MkdirAll(tree, 0o700); err != nil {
		return nil, err
	}
	// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
	gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
		return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
	}
	remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
	if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
		return nil, err
	}
	dirs, err := m.typeScriptModules(ctx, tree)
	if err != nil {
		return nil, err
	}
	res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
	for _, d := range dirs {
		res.Modules = append(res.Modules, d)
		did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
		if err != nil {
			return nil, err
		}
		if refused != "" {
			res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
		}
		if did {
			res.Installed = append(res.Installed, d)
		}
	}
	return res, nil
}

// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
// it, in order.
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
	r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
	if r.Status != 0 || r.Err != "" {
		return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
	}
	var dirs []string
	for _, l := range strings.Split(r.Stdout, "\n") {
		if l = strings.TrimSpace(l); l == "" {
			continue
		}
		raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
		if err != nil {
			return nil, err
		}
		var mj struct {
			Build struct {
				Artifacts []artifact `json:"artifacts"`
			} `json:"build"`
		}
		if json.Unmarshal(raw, &mj) != nil {
			continue // a module.json the build step will refuse by name, not a fetch's business
		}
		for _, a := range mj.Build.Artifacts {
			if a.Language == "typescript" {
				dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
				break
			}
		}
	}
	sort.Strings(dirs)
	return dirs, nil
}

// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
	raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
	if os.IsNotExist(err) {
		return nil, nil, nil
	}
	if err != nil {
		return nil, nil, err
	}
	if err := json.Unmarshal(raw, &pkg); err != nil {
		return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
	}
	deps, _ := pkg["dependencies"].(map[string]any)
	for name := range deps {
		if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
			names = append(names, name)
		}
	}
	sort.Strings(names)
	return names, pkg, nil
}

// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no
// package-lock.json. Empty when it names none or has its lock.
func lockProblem(dir, rel string) (string, error) {
	names, _, err := ownDependencies(dir)
	if err != nil || len(names) == 0 {
		return "", err
	}
	if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
		return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil
	}
	return "", nil
}

// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module
// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and
// the build of that bundle fails naming it.
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) {
	names, pkg, err := ownDependencies(dir)
	if err != nil || len(names) == 0 {
		return false, "", err
	}
	if why, err := lockProblem(dir, rel); why != "" || err != nil {
		return false, why, err
	}
	lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
	if err != nil {
		return false, "", err
	}
	if m.Toolchain == nil || m.Toolchain.Registry == nil {
		return false, "", fmt.Errorf("the module has no package registry to install %s from", rel)
	}
	reg, err := m.Toolchain.Registry()
	if err != nil {
		return false, "", err
	}
	// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
	scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
	if err := os.MkdirAll(scratch, 0o700); err != nil {
		return false, "", err
	}
	for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
		if d, ok := pkg[k].(map[string]any); ok {
			delete(d, sdkPackage)
		}
	}
	delete(pkg, "devDependencies")
	delete(pkg, "scripts")
	raw, _ := json.Marshal(pkg)
	if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
		return false, "", err
	}
	if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
		return false, "", err
	}
	// The registry is given through a config file of its own, never on the command line, and removed after.
	npmrc := filepath.Join(scratch, ".npmrc-install")
	if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
		return false, "", err
	}
	env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
	r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
		"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
	os.Remove(npmrc)
	if why := ranFailed(r); why != "" {
		return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
	}
	nm := filepath.Join(scratch, "node_modules")
	if err := removeSDKCopies(nm); err != nil {
		return false, "", err
	}
	if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
		return false, "", err
	}
	if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
		return false, "", err
	}
	return true, "", os.RemoveAll(scratch)
}

// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
func removeSDKCopies(nm string) error {
	var found []string
	err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
			found = append(found, p)
			return fs.SkipDir
		}
		return nil
	})
	if err != nil {
		return err
	}
	for _, p := range found {
		if err := os.RemoveAll(p); err != nil {
			return err
		}
	}
	return nil
}

// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
func (m *Module) CleanWorkspaces(now time.Time) {
	entries, err := os.ReadDir(m.Work)
	if err != nil {
		return
	}
	for _, e := range entries {
		info, err := e.Info()
		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
			continue
		}
		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
		}
	}
}