323 lines
13 KiB
Go
323 lines
13 KiB
Go
package main
|
|
|
|
// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
|
|
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
|
|
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
|
|
//
|
|
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
|
|
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
|
|
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
|
|
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
|
|
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
|
|
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
|
)
|
|
|
|
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
|
|
const ForgeAlias = "mesh-forge"
|
|
|
|
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
|
const sdkPackage = "@novox/mesh-sdk"
|
|
|
|
// WorkspaceAge is how long a workspace is kept.
|
|
const WorkspaceAge = 24 * time.Hour
|
|
|
|
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
|
|
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
|
|
|
// FetchResult is what a passed fetch step says.
|
|
type FetchResult struct {
|
|
Workspace string `json:"workspace"`
|
|
Commit string `json:"commit"`
|
|
// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
|
|
Modules []string `json:"modules"`
|
|
Installed []string `json:"installed"`
|
|
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
|
|
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
|
|
Refused []Refusal `json:"refused"`
|
|
// The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk
|
|
// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
|
|
// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
|
|
// left at the trunk commit with nothing installed.
|
|
Base string `json:"base,omitempty"`
|
|
Merge string `json:"merge,omitempty"`
|
|
Conflicts []string `json:"conflicts,omitempty"`
|
|
Changed []string `json:"changed,omitempty"`
|
|
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
|
}
|
|
|
|
// Refusal is a module a fetch did not install the packages of.
|
|
type Refusal struct {
|
|
Module string `json:"module"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
func (m *Module) fetchTool() StepTool {
|
|
return StepTool{
|
|
Name: "build_node_fetch",
|
|
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
|
"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
|
|
"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
|
|
Input: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
|
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
|
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
|
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
|
},
|
|
"required": []string{"repository", "commit"},
|
|
},
|
|
Long: true,
|
|
Run: m.fetch,
|
|
}
|
|
}
|
|
|
|
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
|
repository, _ := args["repository"].(string)
|
|
commit, _ := args["commit"].(string)
|
|
ref, _ := args["ref"].(string)
|
|
base, _ := args["base"].(string)
|
|
if base != "" && !commitOK.MatchString(base) {
|
|
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
|
|
}
|
|
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
|
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
|
}
|
|
m.CleanWorkspaces(m.now())
|
|
id := buildstep.NewID()
|
|
root := filepath.Join(m.Work, id)
|
|
tree := filepath.Join(root, "tree")
|
|
if err := os.MkdirAll(tree, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
|
|
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
|
|
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
|
|
}
|
|
remote := m.remote(repository)
|
|
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil {
|
|
return nil, err
|
|
}
|
|
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
|
|
if base != "" {
|
|
if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil {
|
|
return nil, err
|
|
}
|
|
if res.Merge == "" {
|
|
return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
|
|
}
|
|
}
|
|
dirs, err := m.typeScriptModules(ctx, tree)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, d := range dirs {
|
|
res.Modules = append(res.Modules, d)
|
|
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if refused != "" {
|
|
res.Refused = append(res.Refused, Refusal{Module: d, Reason: refused})
|
|
}
|
|
if did {
|
|
res.Installed = append(res.Installed, d)
|
|
}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
|
|
// it, in order.
|
|
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
|
|
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
|
if r.Status != 0 || r.Err != "" {
|
|
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
|
}
|
|
var dirs []string
|
|
for _, l := range strings.Split(r.Stdout, "\n") {
|
|
if l = strings.TrimSpace(l); l == "" {
|
|
continue
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var mj struct {
|
|
Build struct {
|
|
Artifacts []artifact `json:"artifacts"`
|
|
} `json:"build"`
|
|
}
|
|
if json.Unmarshal(raw, &mj) != nil {
|
|
continue // a module.json the build step will refuse by name, not a fetch's business
|
|
}
|
|
for _, a := range mj.Build.Artifacts {
|
|
if a.Language == "typescript" {
|
|
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
|
break
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(dirs)
|
|
return dirs, nil
|
|
}
|
|
|
|
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
|
|
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
|
|
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
|
|
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
|
if os.IsNotExist(err) {
|
|
return nil, nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if err := json.Unmarshal(raw, &pkg); err != nil {
|
|
return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
|
}
|
|
deps, _ := pkg["dependencies"].(map[string]any)
|
|
for name := range deps {
|
|
if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
|
|
names = append(names, name)
|
|
}
|
|
}
|
|
sort.Strings(names)
|
|
return names, pkg, nil
|
|
}
|
|
|
|
// lockProblem says why a module's own packages cannot be installed: it names packages beyond the SDK and has no
|
|
// package-lock.json. Empty when it names none or has its lock.
|
|
func lockProblem(dir, rel string) (string, error) {
|
|
names, _, err := ownDependencies(dir)
|
|
if err != nil || len(names) == 0 {
|
|
return "", err
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "package-lock.json")); err != nil {
|
|
return fmt.Sprintf("%s names packages beyond the SDK and has no package-lock.json (%s): commit one", rel, strings.Join(names, ", ")), nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
// installOwn installs a module's own packages into its directory, or does nothing for a module that has none. A module
|
|
// that names packages and has no lock is not installed and not an error: the reason is answered, the fetch passes and
|
|
// the build of that bundle fails naming it.
|
|
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (installed bool, refused string, err error) {
|
|
names, pkg, err := ownDependencies(dir)
|
|
if err != nil || len(names) == 0 {
|
|
return false, "", err
|
|
}
|
|
if why, err := lockProblem(dir, rel); why != "" || err != nil {
|
|
return false, why, err
|
|
}
|
|
lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
if m.Toolchain == nil || m.Toolchain.Registry == nil {
|
|
return false, "", fmt.Errorf("the module has no package registry to install %s from", rel)
|
|
}
|
|
reg, err := m.Toolchain.Registry()
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
|
|
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
|
|
if err := os.MkdirAll(scratch, 0o700); err != nil {
|
|
return false, "", err
|
|
}
|
|
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
|
|
if d, ok := pkg[k].(map[string]any); ok {
|
|
delete(d, sdkPackage)
|
|
}
|
|
}
|
|
delete(pkg, "devDependencies")
|
|
delete(pkg, "scripts")
|
|
raw, _ := json.Marshal(pkg)
|
|
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
|
|
return false, "", err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
|
|
return false, "", err
|
|
}
|
|
// The registry is given through a config file of its own, never on the command line, and removed after.
|
|
npmrc := filepath.Join(scratch, ".npmrc-install")
|
|
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
|
|
return false, "", err
|
|
}
|
|
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
|
|
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
|
|
"--no-audit", "--no-fund", "--prefix", scratch, "--userconfig", npmrc)
|
|
os.Remove(npmrc)
|
|
if why := ranFailed(r); why != "" {
|
|
return false, "", fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
|
|
}
|
|
nm := filepath.Join(scratch, "node_modules")
|
|
if err := removeSDKCopies(nm); err != nil {
|
|
return false, "", err
|
|
}
|
|
if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
|
|
return false, "", err
|
|
}
|
|
if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
|
|
return false, "", err
|
|
}
|
|
return true, "", os.RemoveAll(scratch)
|
|
}
|
|
|
|
// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
|
|
func removeSDKCopies(nm string) error {
|
|
var found []string
|
|
err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
|
|
found = append(found, p)
|
|
return fs.SkipDir
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, p := range found {
|
|
if err := os.RemoveAll(p); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
|
func (m *Module) CleanWorkspaces(now time.Time) {
|
|
entries, err := os.ReadDir(m.Work)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, e := range entries {
|
|
info, err := e.Info()
|
|
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
|
continue
|
|
}
|
|
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
|
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
|
}
|
|
}
|
|
}
|