Add the Node build module's fetch, test step, registry and toolchain install (issues 1488, 1489, 1491), part 17 of 19
This commit is contained in:
@@ -0,0 +1,263 @@
|
||||
package main
|
||||
|
||||
// The test step (novox/hq ADR 0445 §4; issues 1491 and 1292): the two levels of a TypeScript package's tests, run
|
||||
// against a fetched workspace with the toolchain's tsc. The unit level is the package's type-check and its
|
||||
// package.json `test` script; the integration level is its `test:integration` script, with the type-check as its
|
||||
// setup. Each is timed and judged against its bound; a level over its bound says so in the result and does not fail
|
||||
// (the measure of the catalogue's TypeScript is a later release's, R4f). A package with neither a script nor .ts files
|
||||
// has no tests, and the result says so.
|
||||
//
|
||||
// **The bounds are the Go module's** (5 s for the unit level; 30 s of setup and 60 s of running for the integration
|
||||
// level), because ADR 0445 names no figures for TypeScript. They are constants here, said in every result.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// The bounds of a level (ADR 0445 §4).
|
||||
const (
|
||||
UnitBound = 5 * time.Second
|
||||
IntegrationSetupBound = 30 * time.Second
|
||||
IntegrationRunBound = 60 * time.Second
|
||||
)
|
||||
|
||||
// TestResult is what a test step passes with, with the field names of mesh-build-go's (issue 1290). The step ends
|
||||
// passed whenever the level ran, whatever its verdict, and Passed is the verdict; it ends failed only when the tests
|
||||
// could not be run (no toolchain, a command that cannot start, a workspace that was never fetched).
|
||||
type TestResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Module string `json:"module"`
|
||||
Level string `json:"level"`
|
||||
Passed bool `json:"passed"`
|
||||
// Why says, in words, why the level failed or went over its bound, or that it has no tests.
|
||||
Why string `json:"why,omitempty"`
|
||||
NoTests bool `json:"no_tests,omitempty"`
|
||||
// Ran names what ran: "type-check", "test" or "test:integration".
|
||||
Ran []string `json:"ran,omitempty"`
|
||||
// Elapsed is the level's running time in seconds (at the unit level the type-check and the test script together);
|
||||
// Setup is the integration level's setup, the type-check, in seconds. Bound and SetupBound are the bounds the
|
||||
// level is judged by.
|
||||
Elapsed float64 `json:"elapsed"`
|
||||
Setup float64 `json:"setup,omitempty"`
|
||||
Bound float64 `json:"bound"`
|
||||
SetupBound float64 `json:"setup_bound,omitempty"`
|
||||
// OverBound says a bound was exceeded. Unlike mesh-build-go's, an over-bound level still passes: the measure of
|
||||
// the catalogue's TypeScript against the bounds is a later release's (R4f), not this one's gate.
|
||||
OverBound bool `json:"over_bound"`
|
||||
Failed []Failure `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
// Failure is a check or script that failed; Output is the first lines it wrote.
|
||||
type Failure struct {
|
||||
Package string `json:"package"`
|
||||
Test string `json:"test,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) testTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_test",
|
||||
Description: "Run a TypeScript package's tests in a fetched workspace: the unit level is the type-check (the toolchain's " +
|
||||
"tsc) and the package's test script, the integration level its test:integration script. Each level is timed " +
|
||||
"and judged against its bound (unit 5 s; integration 30 s setup and 60 s running); a level over its bound " +
|
||||
"says so and does not fail. A package with neither script nor .ts files has no tests. Answers a step id at " +
|
||||
"once; the step ends on its key in the steps state.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch step answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the package's directory in the workspace, '.' when it is the root"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}},
|
||||
},
|
||||
"required": []string{"workspace", "level"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.test,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
level, _ := args["level"].(string)
|
||||
if level != "unit" && level != "integration" {
|
||||
return nil, fmt.Errorf("level %q is neither unit nor integration", level)
|
||||
}
|
||||
root, dir, err := m.moduleDirOf(workspace, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scripts, err := packageScripts(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sources, err := typeScriptFiles(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := &TestResult{Workspace: workspace, Module: module, Level: level}
|
||||
script := "test"
|
||||
if level == "integration" {
|
||||
script = "test:integration"
|
||||
}
|
||||
hasScript := scripts[script] != ""
|
||||
if level == "integration" && !hasScript {
|
||||
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script"
|
||||
return res, nil
|
||||
}
|
||||
if !hasScript && len(sources) == 0 {
|
||||
res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files"
|
||||
return res, nil
|
||||
}
|
||||
if err := m.provide(root); err != nil {
|
||||
return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err)
|
||||
}
|
||||
where := module
|
||||
if where == "" {
|
||||
where = "."
|
||||
}
|
||||
var check, run time.Duration
|
||||
var failed []Failure
|
||||
// a command that could not start is not a verdict: the step fails. One that ran and said no is.
|
||||
ran := func(name string, r forgekey.Ran) error {
|
||||
if r.Err != "" {
|
||||
return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr))
|
||||
}
|
||||
if r.Status != 0 {
|
||||
failed = append(failed, Failure{Package: where, Test: name, Output: lines(r.Stdout + "\n" + r.Stderr)})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(sources) > 0 {
|
||||
start := m.now()
|
||||
r := m.runIn(ctx, dir, nil, "node", m.typeCheck(dir, sources)...)
|
||||
check = m.now().Sub(start)
|
||||
if err := ran("type-check", r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Ran = append(res.Ran, "type-check")
|
||||
}
|
||||
if hasScript && len(failed) == 0 {
|
||||
start := m.now()
|
||||
env := []string{"npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm")}
|
||||
r := m.runIn(ctx, dir, env, "npm", scriptArgs(script)...)
|
||||
run = m.now().Sub(start)
|
||||
if err := ran(script, r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Ran = append(res.Ran, script)
|
||||
}
|
||||
res.Failed = failed
|
||||
var why []string
|
||||
for _, f := range failed {
|
||||
why = append(why, fmt.Sprintf("%s failed in %s: %s", f.Test, f.Package, firstLine(f.Output)))
|
||||
}
|
||||
if level == "unit" {
|
||||
res.Elapsed, res.Bound = (check + run).Seconds(), UnitBound.Seconds()
|
||||
res.OverBound = check+run > UnitBound
|
||||
if res.OverBound {
|
||||
why = append(why, fmt.Sprintf("the unit level took %.2f s, over its bound of %s", res.Elapsed, UnitBound))
|
||||
}
|
||||
} else {
|
||||
res.Setup, res.Elapsed = check.Seconds(), run.Seconds()
|
||||
res.SetupBound, res.Bound = IntegrationSetupBound.Seconds(), IntegrationRunBound.Seconds()
|
||||
res.OverBound = check > IntegrationSetupBound || run > IntegrationRunBound
|
||||
if res.OverBound {
|
||||
why = append(why, fmt.Sprintf("the integration level took %.2f s of setup and %.2f s of running, over its bounds of %s and %s",
|
||||
res.Setup, res.Elapsed, IntegrationSetupBound, IntegrationRunBound))
|
||||
}
|
||||
}
|
||||
res.Passed = len(failed) == 0
|
||||
res.Why = strings.Join(why, "; ")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// lines is the first lines of a command's output.
|
||||
func lines(out string) string {
|
||||
var keep []string
|
||||
for _, l := range strings.Split(out, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
keep = append(keep, l)
|
||||
if len(keep) == 15 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(keep, "\n")
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
if i := strings.IndexByte(strings.TrimSpace(s), '\n'); i >= 0 {
|
||||
return strings.TrimSpace(s)[:i]
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// scriptArgs is npm's line for a script: `npm test` for test, `npm run <name>` for another.
|
||||
func scriptArgs(script string) []string {
|
||||
if script == "test" {
|
||||
return []string{"test"}
|
||||
}
|
||||
return []string{"run", script}
|
||||
}
|
||||
|
||||
// typeCheck is tsc's line for the type-check: the package's tsconfig.json when it has one, else the build agent's
|
||||
// flags (toolchain.go:143-147) with --noEmit over every .ts file of the package.
|
||||
func (m *Module) typeCheck(dir string, sources []string) []string {
|
||||
if _, err := os.Stat(filepath.Join(dir, "tsconfig.json")); err == nil {
|
||||
return []string{m.tsc(), "--noEmit", "-p", "."}
|
||||
}
|
||||
return append([]string{m.tsc(), "--noEmit", "--module", "NodeNext", "--moduleResolution", "NodeNext",
|
||||
"--target", "ES2022", "--rootDir", "."}, sources...)
|
||||
}
|
||||
|
||||
// packageScripts are the scripts of the package.json in dir (none without one).
|
||||
func packageScripts(dir string) (map[string]string, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var p struct {
|
||||
Scripts map[string]string `json:"scripts"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return nil, fmt.Errorf("the package.json is not JSON: %w", err)
|
||||
}
|
||||
return p.Scripts, nil
|
||||
}
|
||||
|
||||
// typeScriptFiles are the .ts files of the package (declaration files excepted), relative and sorted, without
|
||||
// node_modules, hidden directories and a build's output.
|
||||
func typeScriptFiles(dir string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := d.Name()
|
||||
if d.IsDir() {
|
||||
if path != dir && (name == "node_modules" || strings.HasPrefix(name, ".")) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") {
|
||||
rel, _ := filepath.Rel(dir, path)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
// The fetch step (novox/hq ADR 0405 §5, ADR 0466 §4 and §5, ADR 0469 §2 and §5; issue 1489): a commit of a repository
|
||||
// of the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and for each
|
||||
// TypeScript module in it the packages its package.json names beyond the SDK, installed from its committed lock.
|
||||
//
|
||||
// **Only what the module itself depends on is installed.** The SDK and nats come with the toolchain, so a bundle that
|
||||
// names only @novox/mesh-sdk (and build-time types and typescript) resolves nothing. One that names more must carry a
|
||||
// package-lock.json, or the step fails naming its directory. The install is the build agent's (mesh-controller
|
||||
// internal/builder/dependencies.go:49-95): in a scratch copy of package.json and the lock with the SDK, devDependencies
|
||||
// and scripts taken out, `npm ci --omit=dev --omit=peer --ignore-scripts`, the install's copies of the SDK removed, and
|
||||
// the result put at the module's node_modules. This module adds --no-bin-links, so the workspace holds no symlink.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1).
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
|
||||
const sdkPackage = "@novox/mesh-sdk"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept.
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// FetchResult is what a passed fetch step says.
|
||||
type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
// Modules are the TypeScript modules' directories found, and Installed those whose own packages were installed.
|
||||
Modules []string `json:"modules"`
|
||||
Installed []string `json:"installed"`
|
||||
}
|
||||
|
||||
func (m *Module) fetchTool() StepTool {
|
||||
return StepTool{
|
||||
Name: "build_node_fetch",
|
||||
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
||||
"workspace of its own, and install each TypeScript module's own packages from its lock (a module naming only the " +
|
||||
"SDK installs nothing). Answers a step id at once; the step ends on its key in the steps state with the workspace's id.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.fetch,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := buildstep.NewID()
|
||||
root := filepath.Join(m.Work, id)
|
||||
tree := filepath.Join(root, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// git and ssh need PATH and the account's HOME, which a clean runner does not supply.
|
||||
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
|
||||
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
|
||||
}
|
||||
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
|
||||
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}}
|
||||
for _, d := range dirs {
|
||||
res.Modules = append(res.Modules, d)
|
||||
did, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if did {
|
||||
res.Installed = append(res.Installed, d)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to
|
||||
// it, in order.
|
||||
func (m *Module) typeScriptModules(ctx context.Context, tree string) ([]string, error) {
|
||||
r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
var dirs []string
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var mj struct {
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if json.Unmarshal(raw, &mj) != nil {
|
||||
continue // a module.json the build step will refuse by name, not a fetch's business
|
||||
}
|
||||
for _, a := range mj.Build.Artifacts {
|
||||
if a.Language == "typescript" {
|
||||
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(dirs)
|
||||
return dirs, nil
|
||||
}
|
||||
|
||||
// ownDependencies are the package.json's dependencies beyond the SDK and beyond what is build-time: the types and
|
||||
// typescript, which the toolchain carries (dependencies.go:49-70). Sorted; none for a module without a package.json.
|
||||
func ownDependencies(dir string) (names []string, pkg map[string]any, err error) {
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "package.json"))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &pkg); err != nil {
|
||||
return nil, nil, fmt.Errorf("the module's package.json is not JSON: %w", err)
|
||||
}
|
||||
deps, _ := pkg["dependencies"].(map[string]any)
|
||||
for name := range deps {
|
||||
if name != sdkPackage && name != "typescript" && !strings.HasPrefix(name, "@types/") {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, pkg, nil
|
||||
}
|
||||
|
||||
// installOwn installs a module's own packages into its directory, or does nothing for a module that has none.
|
||||
func (m *Module) installOwn(ctx context.Context, root, dir, rel string) (bool, error) {
|
||||
names, pkg, err := ownDependencies(dir)
|
||||
if err != nil || len(names) == 0 {
|
||||
return false, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "@novox/") {
|
||||
return false, fmt.Errorf("%s depends on %s, and a fetch holds no credential for the mesh's package registry "+
|
||||
"(ADR 0466 §4), so it cannot be installed", rel, n)
|
||||
}
|
||||
}
|
||||
lock, err := os.ReadFile(filepath.Join(dir, "package-lock.json"))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s names packages beyond the SDK (%s) and has no package-lock.json: commit one, and "+
|
||||
"the step installs exactly what it names", rel, strings.Join(names, ", "))
|
||||
}
|
||||
// A scratch copy, so the module's package.json and lock are never rewritten (dependencies.go installSteps).
|
||||
scratch := filepath.Join(root, "npm-"+strings.NewReplacer("/", "-", ".", "root").Replace(rel))
|
||||
if err := os.MkdirAll(scratch, 0o700); err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, k := range []string{"dependencies", "peerDependencies", "optionalDependencies"} {
|
||||
if d, ok := pkg[k].(map[string]any); ok {
|
||||
delete(d, sdkPackage)
|
||||
}
|
||||
}
|
||||
delete(pkg, "devDependencies")
|
||||
delete(pkg, "scripts")
|
||||
raw, _ := json.Marshal(pkg)
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package.json"), raw, 0o600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(scratch, "package-lock.json"), lock, 0o600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
env := append(m.baseEnv(), "npm_config_cache="+filepath.Join(m.Toolchain.Paths.Cache, "npm"))
|
||||
r := m.Run(ctx, env, "npm", "ci", "--omit=dev", "--omit=peer", "--ignore-scripts", "--no-bin-links",
|
||||
"--no-audit", "--no-fund", "--prefix", scratch)
|
||||
if why := ranFailed(r); why != "" {
|
||||
return false, fmt.Errorf("installing the packages of %s (%s): %s", rel, strings.Join(names, ", "), why)
|
||||
}
|
||||
nm := filepath.Join(scratch, "node_modules")
|
||||
if err := removeSDKCopies(nm); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.Rename(nm, filepath.Join(dir, "node_modules")); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, os.RemoveAll(scratch)
|
||||
}
|
||||
|
||||
// removeSDKCopies removes every copy of the SDK the install pulled in: the toolchain's is the pin.
|
||||
func removeSDKCopies(nm string) error {
|
||||
var found []string
|
||||
err := filepath.WalkDir(nm, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && strings.HasSuffix(filepath.ToSlash(p), "/node_modules/@novox/mesh-sdk") {
|
||||
found = append(found, p)
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range found {
|
||||
if err := os.RemoveAll(p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
// How the toolchain's install reaches the mesh's package registry (novox/hq ADR 0405 §5, ADR 0336; issue 1488).
|
||||
// The lock names @novox/mesh-sdk at the registry's public address, and npm installs a locked tarball only from the
|
||||
// registry it is configured with, so the install is told that registry for the @novox scope and nothing else: every
|
||||
// other package resolves from the public one. This is how today's build agent names it (mesh-controller
|
||||
// internal/builder/dependencies.go: the scope's registry from the npm-package-registry binding, the account and
|
||||
// secret the mesh sealed), read here from the module's own binding and secret files (module.json `binds` and
|
||||
// `secrets`), the same files the lab module reads.
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Scope is the npm scope the mesh's registry answers for.
|
||||
const Scope = "@novox"
|
||||
|
||||
// Registry is the mesh's package registry as the binding names it.
|
||||
type Registry struct {
|
||||
URL, Username, Password string
|
||||
}
|
||||
|
||||
// ReadRegistry reads the npm-package-registry binding and its sealed secret. The registry is the binding's
|
||||
// `public-root` when it has one (the address the lock holds), and otherwise scheme://at:port plus `npm-path`.
|
||||
func ReadRegistry(bindingPath, secretPath string) (Registry, error) {
|
||||
raw, err := os.ReadFile(bindingPath)
|
||||
if err != nil {
|
||||
return Registry{}, fmt.Errorf("no package registry is bound to this module: %w", err)
|
||||
}
|
||||
var b struct {
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves struct {
|
||||
Scheme string `json:"scheme"`
|
||||
Port int `json:"port"`
|
||||
NpmPath string `json:"npm-path"`
|
||||
PublicRoot string `json:"public-root"`
|
||||
} `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &b); err != nil {
|
||||
return Registry{}, fmt.Errorf("the package registry binding is not JSON: %w", err)
|
||||
}
|
||||
url := b.Serves.PublicRoot
|
||||
if url == "" && b.At != "" && b.Serves.NpmPath != "" {
|
||||
scheme := b.Serves.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
url = fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.NpmPath)
|
||||
}
|
||||
if url == "" {
|
||||
return Registry{}, fmt.Errorf("the package registry binding names no address")
|
||||
}
|
||||
if !strings.HasSuffix(url, "/") {
|
||||
url += "/"
|
||||
}
|
||||
secret, err := os.ReadFile(secretPath)
|
||||
if err != nil {
|
||||
return Registry{}, fmt.Errorf("the package registry's secret cannot be read: %w", err)
|
||||
}
|
||||
return Registry{URL: url, Username: b.As, Password: strings.TrimSpace(string(secret))}, nil
|
||||
}
|
||||
|
||||
// Npmrc is the npm configuration text for the install: the @novox scope at the registry, with the account's
|
||||
// credential for it when there is one.
|
||||
func (r Registry) Npmrc() string {
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "%s:registry=%s\n", Scope, r.URL)
|
||||
if r.Username != "" && r.Password != "" {
|
||||
hostPath := r.URL
|
||||
if i := strings.Index(hostPath, "://"); i >= 0 {
|
||||
hostPath = hostPath[i+1:]
|
||||
}
|
||||
fmt.Fprintf(&sb, "%s:_auth=%s\n", hostPath, base64.StdEncoding.EncodeToString([]byte(r.Username+":"+r.Password)))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
// The toolchain directory (novox/hq ADR 0405 §5, ADR 0469 §4; issue 1488): typescript, esbuild, @novox/mesh-sdk and
|
||||
// nats by the versions the module's own lock names, installed by `npm ci --ignore-scripts` into
|
||||
// <cache dir>/toolchain/<recipe version> when that directory is missing. No bin links are made (--no-bin-links): the
|
||||
// toolchain holds no symlink, and tsc and esbuild are run by their real paths. A change of the lock, of the Node
|
||||
// version or of the recipe moves the recipe version and so installs a fresh directory beside the old one; steps use the one named
|
||||
// by the current recipe.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// installedMark is written last into a toolchain directory, so a directory an install did not finish is not taken
|
||||
// for a toolchain.
|
||||
const installedMark = ".installed"
|
||||
|
||||
// Paths are the module's places on its machine.
|
||||
type Paths struct {
|
||||
// Home is the account's home (the state directory): where the mesh writes toolchain/package.json and
|
||||
// toolchain/package-lock.json (module.json resources) and the registry binding and secret.
|
||||
Home string
|
||||
// Cache is the cache directory, outside the home (ADR 0466 §1).
|
||||
Cache string
|
||||
}
|
||||
|
||||
// LockPath is the committed lock file the mesh writes into the home.
|
||||
func (p Paths) LockPath() string { return filepath.Join(p.Home, "toolchain", "package-lock.json") }
|
||||
|
||||
// Dir is the toolchain directory of a recipe version.
|
||||
func (p Paths) Dir(version string) string { return filepath.Join(p.Cache, "toolchain", version) }
|
||||
|
||||
// Toolchain installs the toolchain directories.
|
||||
type Toolchain struct {
|
||||
Paths Paths
|
||||
Run forgekey.Runner
|
||||
Registry func() (Registry, error)
|
||||
}
|
||||
|
||||
// Present reports whether the recipe version's toolchain directory is installed.
|
||||
func (t Toolchain) Present(version string) bool {
|
||||
_, err := os.Stat(filepath.Join(t.Paths.Dir(version), installedMark))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Ensure installs the recipe version's toolchain directory unless it is there: the module's package.json and lock
|
||||
// are copied in, and `npm ci --ignore-scripts` runs once, the mesh's registry named for the @novox scope. A directory
|
||||
// an earlier install left unfinished is removed first.
|
||||
func (t Toolchain) Ensure(ctx context.Context, version string) error {
|
||||
if t.Present(version) {
|
||||
return nil
|
||||
}
|
||||
dir := t.Paths.Dir(version)
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range []string{"package.json", "package-lock.json"} {
|
||||
raw, err := os.ReadFile(filepath.Join(t.Paths.Home, "toolchain", name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the toolchain's %s cannot be read: %w", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), raw, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
reg, err := t.Registry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
npmrc := filepath.Join(dir, ".npmrc-install")
|
||||
if err := os.WriteFile(npmrc, []byte(reg.Npmrc()), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(npmrc)
|
||||
ran := t.Run(ctx, []string{"PATH=" + pathOrDefault(), "HOME=" + t.Paths.Home, "LC_ALL=C", "npm_config_cache=" + filepath.Join(t.Paths.Cache, "npm")},
|
||||
"npm", "ci", "--ignore-scripts", "--no-bin-links", "--no-audit", "--no-fund", "--prefix", dir, "--userconfig", npmrc)
|
||||
if ran.Err != "" || ran.Status != 0 {
|
||||
return fmt.Errorf("npm ci --ignore-scripts for the toolchain %s failed (status %d %s): %s",
|
||||
version, ran.Status, ran.Err, tail(ran.Stderr))
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, installedMark), []byte(version+"\n"), 0o600)
|
||||
}
|
||||
|
||||
func tail(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) > 400 {
|
||||
s = s[len(s)-400:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (t *Toolchain) homeOrEmpty() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.Paths.Home
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The tools this module serves. A tool is named as the module is: build_node_ and its verb (a tool name has no dot).
|
||||
const (
|
||||
StatusTool = "build_node_status"
|
||||
HealthTool = "build_node_health"
|
||||
)
|
||||
|
||||
// StepTool is one step tool of the module: fetch, test or build. Long is a step that answers at once and
|
||||
// ends on its key (ADR 0469 §2); the others answer in their call. Both run behind the toolchain guard.
|
||||
type StepTool struct {
|
||||
Name string
|
||||
Description string
|
||||
Input map[string]any
|
||||
Long bool
|
||||
Run func(ctx context.Context, args map[string]any) (any, error)
|
||||
}
|
||||
|
||||
// StepTools are the module's step tools. Each step's issue adds its own.
|
||||
func (m *Module) StepTools() []StepTool {
|
||||
return []StepTool{m.fetchTool(), m.testTool(), m.buildTool()}
|
||||
}
|
||||
|
||||
// Tools is every tool the module serves: its step tools behind the guard, its status and its health.
|
||||
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools := []stdio.Tool{}
|
||||
for _, s := range steps {
|
||||
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
|
||||
if s.Long {
|
||||
t.Run = m.Long(s.Name, s.Run)
|
||||
} else {
|
||||
t.Run = m.Short(s.Run)
|
||||
}
|
||||
tools = append(tools, t)
|
||||
}
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " +
|
||||
"the recipe version, the toolchain directory and whether it is installed, and why it refuses to build if it does. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }},
|
||||
stdio.Tool{Name: HealthTool,
|
||||
Description: "How the Node build module is: unhealthy while the Node installed on this machine is not the version its " +
|
||||
"recipe builds with, saying both versions; healthy otherwise. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }},
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user