diff --git a/modules/mesh-build-node/.gitignore b/modules/mesh-build-node/.gitignore index f7604aaaa..ce26533a5 100644 --- a/modules/mesh-build-node/.gitignore +++ b/modules/mesh-build-node/.gitignore @@ -1 +1,3 @@ -mesh-build-node +# the binary, built at the module or in its command directory; never the command directory itself +/mesh-build-node +/cmd/mesh-build-node/mesh-build-node diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go index 4627c2c7a..086bd7c01 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -13,6 +13,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "sort" "strings" @@ -37,6 +38,27 @@ type artifact struct { Language string `json:"language"` Entrypoints []string `json:"entrypoints"` External []string `json:"external"` + // Paths are the directories of a package artifact, relative to the module's: one, holding its package.json and + // its package-lock.json (as a Go package's, mesh-build-go gopackage.go); none means the module's directory. + Paths []string `json:"paths"` +} + +// packageDir is the directory of a package artifact relative to its module's directory in dir: the one path it names +// when that is a directory of the module holding a package.json (the contracts folders: ts/seats//, each with its +// own lock, ADR 0634 section 8), else the module's directory itself (a package that names no path, or source paths +// beside the module's package.json: today's form). +func (a artifact) packageDir(dir string) string { + if len(a.Paths) != 1 { + return "." + } + p := path.Clean(strings.TrimSuffix(a.Paths[0], "/")) + if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") { + return "." + } + if _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(p), "package.json")); err != nil { + return "." + } + return p } // BuildResult is what a passed build step says. @@ -88,7 +110,8 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { return nil, err } if a.Kind == "package" { - return m.buildPackage(ctx, workspace, module, root, dir, a, publish) + at := a.packageDir(dir) + return m.buildPackage(ctx, workspace, module, root, filepath.Join(dir, filepath.FromSlash(at)), at, a, publish) } if why, err := lockProblem(dir, module); err != nil || why != "" { if err == nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/check.go b/modules/mesh-build-node/cmd/mesh-build-node/check.go index 651688ac4..cda4b1ca5 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/check.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/check.go @@ -16,6 +16,7 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "strings" "time" @@ -99,78 +100,89 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } - scripts, err := packageScripts(dir) - if err != nil { - return nil, err - } - sources, err := typeScriptFiles(dir) - if err != nil { - return nil, err - } res := &TestResult{Workspace: workspace, Module: module, Level: level, By: sdkstep.By("mesh-build-node")} script := "test" if level == "integration" { script = "test:integration" } - hasScript := scripts[script] != "" - if level == "integration" && !hasScript { - res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test:integration script" - return res, nil - } - if !hasScript && len(sources) == 0 { - res.Passed, res.NoTests, res.Why = true, true, "no tests: the package has no test script and no .ts files" - return res, nil - } - if err := m.provide(root); err != nil { - return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err) - } where := module if where == "" { where = "." } + targets, err := testTargets(dir, where) + if err != nil { + return nil, err + } var check, run time.Duration var failed []Failure - // a command that could not start is not a verdict: the step fails. One that ran and said no is. - ran := func(name string, r forgekey.Ran, report string) error { - if r.Err != "" { - return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) - } - if r.Status != 0 { - failed = append(failed, testFailures(where, name, dir, r, report)...) - } - return nil - } - if len(sources) > 0 { - // A package that builds (a build or prepare script, which the fetch's --ignore-scripts never runs) is emitted - // by tsc into the workspace, shut off like every run step, so its tests find what they import of its own output - // (issue 1869: mesh-sdk's tests import ../dist); any other is type-checked without emitting. - name, line := "type-check", m.typeCheck(dir, sources) - if emit := m.emitBuild(dir, scripts); emit != nil { - name, line = "build", emit - } - start := m.now() - r := m.runIn(ctx, root, false, dir, nil, "node", line...) - check = m.now().Sub(start) - if err := ran(name, r, ""); err != nil { + tested := false + for _, tg := range targets { + scripts, err := packageScripts(tg.dir) + if err != nil { return nil, err } - res.Ran = append(res.Ran, name) - } - if hasScript && len(failed) == 0 { - start := m.now() - // node --test writes its TAP report to a file of the workspace (issue 1901), whatever the script prints itself - report := filepath.Join(root, ".mesh-node-test-"+level+".tap") - _ = os.Remove(report) - env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), - "NODE_OPTIONS=" + buildstep.NodeTestOptions(report)} - r := m.runIn(ctx, root, false, dir, env, "npm", scriptArgs(script)...) - run = m.now().Sub(start) - tap, _ := os.ReadFile(report) - _ = os.Remove(report) - if err := ran(script, r, string(tap)); err != nil { + sources, err := typeScriptFiles(tg.dir) + if err != nil { return nil, err } - res.Ran = append(res.Ran, script) + hasScript := scripts[script] != "" + if (level == "integration" && !hasScript) || (!hasScript && len(sources) == 0) { + continue + } + tested = true + if err := m.provide(root); err != nil { + return nil, fmt.Errorf("providing the toolchain's packages to the workspace: %w", err) + } + // a command that could not start is not a verdict: the step fails. One that ran and said no is. + ran := func(name string, r forgekey.Ran, report string) error { + if r.Err != "" { + return fmt.Errorf("%s could not be run: status %d %s: %s", name, r.Status, r.Err, firstLine(r.Stderr)) + } + if r.Status != 0 { + failed = append(failed, testFailures(tg.where, name, tg.dir, r, report)...) + } + return nil + } + if len(sources) > 0 { + // A package that builds (a build or prepare script, which the fetch's --ignore-scripts never runs) is emitted + // by tsc into the workspace, shut off like every run step, so its tests find what they import of its own output + // (issue 1869: mesh-sdk's tests import ../dist); any other is type-checked without emitting. + name, line := "type-check", m.typeCheck(tg.dir, sources) + if emit := m.emitBuild(tg.dir, scripts); emit != nil { + name, line = "build", emit + } + start := m.now() + r := m.runIn(ctx, root, false, tg.dir, nil, "node", line...) + check += m.now().Sub(start) + if err := ran(name, r, ""); err != nil { + return nil, err + } + res.Ran = appendOnce(res.Ran, name) + } + if hasScript && len(failed) == 0 { + start := m.now() + // node --test writes its TAP report to a file of the workspace (issue 1901), whatever the script prints itself + report := filepath.Join(root, ".mesh-node-test-"+level+".tap") + _ = os.Remove(report) + env := []string{"PATH=" + binDir(root) + ":" + pathOrDefault(), "npm_config_cache=" + filepath.Join(m.Toolchain.Paths.Cache, "npm"), + "NODE_OPTIONS=" + buildstep.NodeTestOptions(report)} + r := m.runIn(ctx, root, false, tg.dir, env, "npm", scriptArgs(script)...) + run += m.now().Sub(start) + tap, _ := os.ReadFile(report) + _ = os.Remove(report) + if err := ran(script, r, string(tap)); err != nil { + return nil, err + } + res.Ran = appendOnce(res.Ran, script) + } + } + if !tested { + res.Passed, res.NoTests = true, true + res.Why = "no tests: the package has no test script and no .ts files" + if level == "integration" { + res.Why = "no tests: the package has no test:integration script" + } + return res, nil } res.Failed = failed var why []string @@ -294,3 +306,56 @@ func typeScriptFiles(dir string) ([]string, error) { }) return out, err } + +// testTarget is a directory whose tests a level runs, and the name it is reported under. +type testTarget struct{ dir, where string } + +// testTargets are the directories a module's tests run in: each TypeScript package artifact's own path when every +// package the module declares names one (ADR 0634 §8: its package.json, its lock, its tests), else the module's +// directory. +func testTargets(dir, where string) ([]testTarget, error) { + whole := []testTarget{{dir, where}} + raw, err := os.ReadFile(filepath.Join(dir, "module.json")) + if err != nil { + return whole, nil + } + var mj struct { + Build struct { + Artifacts []artifact `json:"artifacts"` + } `json:"build"` + } + if json.Unmarshal(raw, &mj) != nil { + return whole, nil + } + var out []testTarget + seen := map[string]bool{} + for _, a := range mj.Build.Artifacts { + if a.Language != "typescript" { + continue + } + if a.Kind != "package" { + return whole, nil // a bundle is tested in its module's directory + } + at := a.packageDir(dir) + if at == "." { + return whole, nil + } + if !seen[at] { + seen[at] = true + out = append(out, testTarget{filepath.Join(dir, filepath.FromSlash(at)), path.Join(where, at)}) + } + } + if len(out) == 0 { + return whole, nil + } + return out, nil +} + +func appendOnce(list []string, s string) []string { + for _, x := range list { + if x == s { + return list + } + } + return append(list, s) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index e758fab70..bd6343ecd 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -17,6 +17,7 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "regexp" "sort" @@ -210,8 +211,8 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group return res, nil } -// typeScriptModules are the directories of the tree whose module.json declares a typescript artifact, relative to -// it, in order. +// typeScriptModules are the directories of the tree that a typescript artifact of a module.json is built in, relative to +// it, in order: the module's directory for a bundle, the artifact's path for a package. func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []string, packages map[string]bool, err error) { r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json") if r.Status != 0 || r.Err != "" { @@ -237,13 +238,21 @@ func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []str if json.Unmarshal(raw, &mj) != nil { continue // a module.json the build step will refuse by name, not a fetch's business } + // A bundle is built in its module's directory, a package in its own path (ADR 0634 §8: as at a repository's + // root, each package with its own lock); the first artifact to claim a directory decides it. for _, a := range mj.Build.Artifacts { - if a.Language == "typescript" { - d := filepath.ToSlash(filepath.Dir(l)) - dirs = append(dirs, d) - packages[d] = a.Kind == "package" - break + if a.Language != "typescript" { + continue } + d := filepath.ToSlash(filepath.Dir(l)) + if a.Kind == "package" { + d = path.Join(d, a.packageDir(filepath.Join(tree, filepath.FromSlash(d)))) + } + if _, seen := packages[d]; seen { + continue + } + dirs = append(dirs, d) + packages[d] = a.Kind == "package" } } sort.Strings(dirs) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index 0663f180a..713d76c6c 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -469,11 +469,13 @@ func fetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, // (publish) publishes it, from this module's own process with the registry's write credential, which no run step is // given: a version the registry lacks is published; the same packed file already there is done without a publish; // another packed file under that version fails `version taken:`. The reference is @, answered only when -// the registry holds this build's file; a candidate answers its digest alone. -func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir string, a artifact, publish bool) (any, error) { - rel := module - if rel == "" { - rel = "." +// the registry holds this build's file; a candidate answers its digest alone. dir is the package's directory, at its +// place in the module's. +func (m *Module) buildPackage(ctx context.Context, workspace, module, root, dir, at string, a artifact, publish bool) (any, error) { + // rel is the package's directory in the workspace, as the fetch installed it (the shims lie under it). + rel := path.Join(module, at) + if module == "" { + rel = path.Join(".", at) } p, err := m.buildAndPack(ctx, root, dir, rel, publish) if err != nil { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go b/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go new file mode 100644 index 000000000..98dd5f781 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/package_paths_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" +) + +// A folder module (mesh-contracts' root, a domain folder) declares several package artifacts, each at its own path +// with its own package.json and package-lock.json and none at the module's directory (ADR 0634 section 8). +const folderManifest = `{"module":"c","build":{"artifacts":[` + + `{"name":"ts-a","kind":"package","language":"typescript","paths":["ts/seats/a"]},` + + `{"name":"ts-b","kind":"package","language":"typescript","paths":["ts/seats/b/"]}]}}` + +func folderFiles(prefix string) map[string]string { + files := map[string]string{prefix + "module.json": folderManifest} + for _, s := range []string{"a", "b"} { + files[prefix+"ts/seats/"+s+"/package.json"] = `{"name":"@x/` + s + `","version":"0.1.0","scripts":{"build":"tsc","test":"node --test"}}` + files[prefix+"ts/seats/"+s+"/package-lock.json"] = `{"lockfileVersion":3}` + files[prefix+"ts/seats/"+s+"/index.ts"] = "export const v = 1;\n" + } + return files +} + +// The fetch installs each package artifact at its path from the lock there, and finds no lock missing at the module. +func TestAFolderModulesPackagesAreInstalledEachAtItsPath(t *testing.T) { + m, f := fetchMachine(t, folderFiles("modules/c/"), nil) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + if len(r.Modules) != 2 || r.Modules[0] != "modules/c/ts/seats/a" || r.Modules[1] != "modules/c/ts/seats/b" || len(r.Installed) != 2 { + t.Errorf("result = %+v, want both package directories found and installed", r) + } + var prefixes []string + for _, c := range npmCalls(f) { + for i, a := range c.args { + if a == "--prefix" { + prefixes = append(prefixes, strings.TrimPrefix(c.args[i+1], filepath.Join(m.Work, r.Workspace, "tree"))) + } + } + } + if len(prefixes) != 2 || prefixes[0] != "/modules/c/ts/seats/a" || prefixes[1] != "/modules/c/ts/seats/b" { + t.Errorf("npm ci ran with --prefix %v, want each package's own directory", prefixes) + } +} + +// The build runs the package's script and pack in its own path, and the test level runs in each path. +func TestAFolderModulesPackageIsBuiltAndTestedInItsPath(t *testing.T) { + m, f, work := newMachine(t) + workspaceWith(t, work, folderFiles("")) + f.effect = func(c call) { + if c.name == "npm" && len(c.args) > 0 && c.args[0] == "pack" { + for i, a := range c.args { + if a == "--pack-destination" { + write(t, filepath.Join(c.args[i+1], "x.tgz"), "packed", 0o644) + } + } + } + } + if _, err := m.build(context.Background(), map[string]any{"workspace": "w1", "module": ".", "artifact": "ts-b"}); err != nil { + t.Fatal(err) + } + want := filepath.Join(work, "w1", "tree", "ts", "seats", "b") + built := 0 + for _, c := range f.calls { + if c.name == "npm" { + built++ + if c.dir != want { + t.Errorf("%s ran in %s, want %s", joined(c), c.dir, want) + } + } + } + if built != 2 { + t.Errorf("%d npm commands, want the build and the pack", built) + } + f.calls = nil + res, err := m.test(context.Background(), testArgs("unit")) + if err != nil { + t.Fatal(err) + } + tests := map[string]bool{} + for _, c := range f.calls { + if c.name == "npm" { + tests[c.dir] = true + } + } + if r := res.(*TestResult); !r.Passed || len(tests) != 2 || !tests[want] || !tests[filepath.Join(work, "w1", "tree", "ts", "seats", "a")] { + t.Errorf("unit level = %+v, ran tests in %v, want both package paths", r, tests) + } +}