Add the Node build module's fetch, test step, registry and toolchain install (issues 1488, 1489, 1491), part 17 of 19
This commit is contained in:
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
|
||||
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
|
||||
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
|
||||
//
|
||||
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
|
||||
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
|
||||
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
|
||||
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
|
||||
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
|
||||
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// FetchResult is what a passed fetch step says.
|
||||
type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
|
||||
Modules []string `json:"modules"`
|
||||
Installed []string `json:"installed"`
|
||||
}
|
||||
|
||||
func (m *Module) fetchTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_fetch",
|
||||
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
||||
"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
|
||||
"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.fetch,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
|
||||
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
|
||||
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}}
|
||||
for _, d := range dirs {
|
||||
res.Modules = append(res.Modules, d)
|
||||
did, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if did {
|
||||
res.Installed = append(res.Installed, d)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
|
||||
// it, in order.
|
||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
|
||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
var dirs []string
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var mj struct {
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if json.Unmarshal(raw, &mj) != nil {
|
||||
continue // a module.json the build step will refuse by name, not a fetch's business
|
||||
}
|
||||
for _, a := range mj.Build.Artifacts {
|
||||
if a.Language == "typescript" {
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
|
||||
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
|
||||
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &pkg); err != nil {
|
||||
return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||
}
|
||||
deps, _ := pkg["dependencies"].(map[string]any)
|
||||
for name := range deps {
|
||||
if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, pkg, nil
|
||||
}
|
||||
|
||||
// installOwn installs a module's own packages into its directory, or does nothing for a module that has none.
|
||||
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (bool, error) {
|
||||
names, pkg, err := ownDependencies(dir)
|
||||
if err != nil || len(names) == 0 {
|
||||
return false, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "@novox/") {
|
||||
return false, fmt.Errorf("%s depends on %s, and a fetch holds no credential for the mesh's package registry "+
|
||||
"(ADR 0466 §4), so it cannot be installed", rel, n)
|
||||
}
|
||||
}
|
||||
lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s names packages beyond the SDK (%s) and has no package-lock.json: commit one, and "+
|
||||
"the step installs exactly what it names", rel, strings.Join(names, ", "))
|
||||
}
|
||||
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
|
||||
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
|
||||
if err := os.MkdirAll(scratch, 0o700); err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
|
||||
if d, ok := pkg[k].(map[string]any); ok {
|
||||
delete(d, sdkPackage)
|
||||
}
|
||||
}
|
||||
delete(pkg, "devDependencies")
|
||||
delete(pkg, "scripts")
|
||||
raw, _ := json.Marshal(pkg)
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
|
||||
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
|
||||
"--no-audit", "--no-fund", "--prefix", scratch)
|
||||
if why := ranFailed(r); why != "" {
|
||||
return false, fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
|
||||
}
|
||||
nm := filepath.Join(scratch, "node_modules")
|
||||
if err := removeSDKCopies(nm); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, os.RemoveAll(scratch)
|
||||
}
|
||||
|
||||
// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
|
||||
func removeSDKCopies(nm string) error {
|
||||
var found []string
|
||||
err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
|
||||
found = append(found, p)
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range found {
|
||||
if err := os.RemoveAll(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user