mesh-build-node: narrow a step's workspace as H-A-136 and H-A-137 say; a narrowed-out package is no module of the step #716

Merged
mesh-tools merged 2 commits from fix/47-H1-mesh-contracts-publish into main 2026-11-20 15:12:44 +00:00
9 changed files with 325 additions and 22 deletions
@@ -44,16 +44,37 @@ type artifact struct {
Paths []string `json:"paths"`
}
// namedPath is the one path a package artifact names, cleaned, relative to its module's directory; false when it names
// none, several, or one outside the module.
func (a artifact) namedPath() (string, bool) {
if len(a.Paths) != 1 {
return "", false
}
p := path.Clean(strings.TrimSuffix(a.Paths[0], "/"))
if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") {
return "", false
}
return p, true
}
// narrowedAway says whether the package's one named path is absent from the module's directory dir: in a workspace
// narrowed to another artifact's paths, it is not this build's.
func (a artifact) narrowedAway(dir string) bool {
p, ok := a.namedPath()
if !ok {
return false
}
_, err := os.Lstat(filepath.Join(dir, filepath.FromSlash(p)))
return os.IsNotExist(err)
}
// packageDir is the directory of a package artifact relative to its module's directory in dir: the one path it names
// when that is a directory of the module holding a package.json (the contracts folders: ts/seats/<seat>/, each with its
// own lock, ADR 0634 section 8), else the module's directory itself (a package that names no path, or source paths
// beside the module's package.json: today's form).
func (a artifact) packageDir(dir string) string {
if len(a.Paths) != 1 {
return "."
}
p := path.Clean(strings.TrimSuffix(a.Paths[0], "/"))
if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") {
p, ok := a.namedPath()
if !ok {
return "."
}
if _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(p), "package.json")); err != nil {
@@ -135,7 +135,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul
if where == "" {
where = "."
}
targets, err := testTargets(dir, where)
targets, err := testTargets(dir, where, narrowed(filepath.Join(root, "tree")))
if err != nil {
return nil, err
}
@@ -338,8 +338,9 @@ type testTarget struct{ dir, where string }
// testTargets are the directories a module's tests run in: each TypeScript package artifact's own path when every
// package the module declares names one (ADR 0634 §8: its package.json, its lock, its tests), else the module's
// directory.
func testTargets(dir, where string) ([]testTarget, error) {
// directory. In a workspace narrowed to one artifact's paths (narrowedTree), a package whose path was left out is no
// target of this step (ADR 0499 clause 4; design 47 H, H-A-136).
func testTargets(dir, where string, narrowedTree bool) ([]testTarget, error) {
whole := []testTarget{{dir, where}}
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
if err != nil {
@@ -362,6 +363,9 @@ func testTargets(dir, where string) ([]testTarget, error) {
if a.Kind != "package" {
return whole, nil // a bundle is tested in its module's directory
}
if narrowedTree && a.narrowedAway(dir) {
continue
}
at := a.packageDir(dir)
if at == "." {
return whole, nil
@@ -175,7 +175,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
}
}
if len(req.Paths) > 0 {
if err := steprun.Narrow(tree, req); err != nil {
if err := narrowTree(tree, req); err != nil { // with the generated-from directories H-A-136 and H-A-137 keep
return nil, err
}
// Said beside the tree, so the listing of the tree's modules leaves out those narrowed away.
@@ -300,6 +300,11 @@ func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []str
continue
}
d := filepath.ToSlash(filepath.Dir(l))
if a.Kind == "package" && narrowed(tree) && a.narrowedAway(filepath.Join(tree, filepath.FromSlash(d))) {
// A package of the module whose own path the narrowing left out (ADR 0499 clause 4; design 47 H, H-A-136:
// nothing else is fetched): no module of this build, never the module's directory, which holds no lock.
continue
}
if a.Kind == "package" {
d = path.Join(d, a.packageDir(filepath.Join(tree, filepath.FromSlash(d))))
}
@@ -22,7 +22,7 @@ func (m *Module) lintStep(ctx context.Context, args map[string]any) (any, error)
if err != nil {
return nil, err
}
targets, err := testTargets(mdir, filepath.ToSlash(filepath.Clean(dir)))
targets, err := testTargets(mdir, filepath.ToSlash(filepath.Clean(dir)), narrowed(filepath.Join(root, "tree")))
if err != nil {
return nil, err
}
@@ -0,0 +1,135 @@
package main
// A copy of mesh-build-go's narrowTree and generatedFromDirs (mesh-build-go/cmd/mesh-build-go/narrow.go) without its Go
// part (the directories of the repository's packages a go.mod requires at the zero version). Copied, not moved to
// mesh-delivery-sdk's steprun beside Narrow: the move is a change to two more repositories (mesh-delivery and back
// into mesh-build-go) and a group delivery for some sixty lines, in a fix that unblocks the build modules' steps now;
// the copy is said here so the move is not forgotten.
import (
"os"
"path"
"path/filepath"
"slices"
"strings"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// narrowedKeptAtRoot are what steprun.Narrow keeps beside the named paths, read from the request's directory: the
// manifest and its lock files. A request narrowed from the repository's root keeps them at the root too, where the
// asked module names none of them; they are removed again.
var narrowedKeptAtRoot = []string{"module.json", "go.sum", "package-lock.json"}
// narrowTree narrows the fetched tree for a step of an artifact that names its paths (ADR 0499 clause 4,
// steprun.Narrow). Beside the artifact's paths and the module's manifest, the workspace holds every generated-from
// directory an artifact of the module names, so the unit level's generator check (steprun.OutOfStep) finds the seat
// definitions and every generated package; nothing else (design 47 H, lead answers H-A-136 and H-A-137).
func narrowTree(tree string, req sdkstep.FetchRequest) error {
dir := cleanDir(req.Dir)
if dir == "" {
return steprun.Narrow(tree, req) // refused there: named paths without the module's directory
}
var outside []string
inside := slices.Clone(req.Paths)
for _, e := range generatedFromDirs(tree, dir) {
if rel, ok := under(dir, e); ok {
if !slices.Contains(inside, rel) {
inside = append(inside, rel)
}
} else if !slices.Contains(outside, e) {
outside = append(outside, e)
}
}
if len(outside) == 0 {
r := req
r.Paths = inside
return steprun.Narrow(tree, r)
}
// A directory outside the module's own: the request is narrowed from the root, with every path the module's
// narrowing keeps named by its path from the root (the manifest and lock files only where the module has them).
all := []string{}
for _, p := range inside {
all = append(all, path.Join(dir, p))
}
for _, f := range narrowedKeptAtRoot {
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(path.Join(dir, f)))); err == nil {
all = append(all, path.Join(dir, f))
}
}
all = append(all, outside...)
if err := steprun.Narrow(tree, sdkstep.FetchRequest{Repository: req.Repository, Commit: req.Commit, Paths: all}); err != nil {
return err
}
for _, f := range narrowedKeptAtRoot {
if !slices.Contains(all, f) {
if err := os.RemoveAll(filepath.Join(tree, f)); err != nil {
return err
}
}
}
return nil
}
func cleanDir(dir string) string {
if strings.TrimSpace(dir) == "" {
return ""
}
return strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/")
}
// under answers p's path relative to dir when p is dir or lies under it.
func under(dir, p string) (string, bool) {
if dir == "." {
return p, true
}
if p == dir {
return ".", true
}
if rel, ok := strings.CutPrefix(p, dir+"/"); ok {
return rel, true
}
return "", false
}
// generatedFromDirs are the generated-from directories (from the tree's root) the artifacts of the module in dir name
// (H-A-136). A module at the repository's root keeps not its generated-from directory, which would be the whole
// repository, but the generator's inputs and outputs: the seats/ directory of each generated-from directory and the
// paths of every artifact of the module naming generated-from (H-A-137). A manifest that is absent or unreadable names
// none; the steps that read it say so.
func generatedFromDirs(tree, dir string) []string {
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(dir), "module.json"))
if err != nil {
return nil
}
man, err := catalogue.Decode(raw)
if err != nil || man.Build == nil {
return nil
}
var out []string
add := func(p string) {
if p == ".." || strings.HasPrefix(p, "../") || slices.Contains(out, p) {
return
}
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(p))); err == nil { // steprun.Narrow refuses a path the tree lacks
out = append(out, p)
}
}
for _, a := range man.Build.Artifacts {
g := strings.TrimSpace(a.GeneratedFrom)
if g == "" {
continue
}
if dir != "." {
add(path.Join(dir, filepath.ToSlash(g)))
continue
}
add(path.Join(filepath.ToSlash(g), "seats"))
for _, p := range a.Paths {
add(path.Clean(filepath.ToSlash(p)))
}
}
return out
}
@@ -0,0 +1,74 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatgen"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
)
// A root contracts module (mesh-contracts at 87eab81, pipeline main.3 of 2026-11-20): two seats, each generated as a Go
// and a TypeScript package naming generated-from ".", in step with the seat generator, beside what no step reads.
func rootContracts(t *testing.T) string {
t.Helper()
tree := t.TempDir()
put := func(p, body string) {
write(t, filepath.Join(tree, filepath.FromSlash(p)), body, 0o644)
}
var arts []string
for _, s := range []string{"one-store", "two-store"} {
put("seats/"+s+"/seat.json", `{"format":1,"seat":"`+s+`","scope":"mesh","grain":"whole","domain":"Module","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"},"answer":{"type":"object"}}]}`)
pkg := strings.ReplaceAll(s, "-", "")
arts = append(arts,
`{"name":"go-`+pkg+`","kind":"package","language":"go","paths":["go/seats/`+pkg+`"],"generated-from":"."}`,
`{"name":"ts-`+pkg+`","kind":"package","language":"typescript","paths":["ts/seats/`+pkg+`"],"generated-from":"."}`)
}
put("module.json", `{"module":"c","version":"0.1","build":{"artifacts":[`+strings.Join(arts, ",")+`]}}`)
put("README.md", "the repository\n")
g, err := seatgen.Load(tree, "example.org/c/go", "@example/c")
if err != nil {
t.Fatal(err)
}
files, err := seatgen.Generate(g)
if err != nil {
t.Fatal(err)
}
for p, body := range files {
put(p, string(body))
}
return tree
}
// Design 47 H, H-A-136 and H-A-137: a step's workspace of one package of a root module keeps, beside the package's path
// and module.json, the seats/ directory and the paths of every artifact naming generated-from, so the unit level's
// generator check finds nothing out of step; nothing else is kept. Narrowed bare (steprun.Narrow), the check names the
// generated files left out.
func TestANarrowedRootContractsModuleIsInStepWithTheGenerator(t *testing.T) {
req := sdkstep.FetchRequest{Repository: "novox/c", Commit: commitSHA, Dir: ".", Paths: []string{"ts/seats/onestore"}}
bare := rootContracts(t)
if err := steprun.Narrow(bare, req); err != nil {
t.Fatal(err)
}
if out, err := steprun.OutOfStep(bare, steprun.GeneratedRootTS); err != nil || len(out) == 0 {
t.Fatalf("narrowed bare: out of step %v (%v), want the left-out packages named", out, err)
}
tree := rootContracts(t)
if err := narrowTree(tree, req); err != nil {
t.Fatal(err)
}
if out, err := steprun.OutOfStep(tree, steprun.GeneratedRootTS); err != nil || len(out) != 0 {
t.Errorf("narrowed: out of step %v (%v), want none", out, err)
}
for _, kept := range []string{"module.json", "seats/two-store/seat.json", "ts/seats/twostore/package.json", "go/seats/twostore/go.mod"} {
if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(kept))); err != nil {
t.Errorf("%s is not kept: %v", kept, err)
}
}
if _, err := os.Stat(filepath.Join(tree, "README.md")); err == nil {
t.Error("README.md is still in the workspace")
}
}
@@ -91,3 +91,54 @@ func TestAFolderModulesPackageIsBuiltAndTestedInItsPath(t *testing.T) {
t.Errorf("unit level = %+v, ran tests in %v, want both package paths", r, tests)
}
}
// A step of one package of a root module with several (mesh-contracts' ts-meshbroker beside ts-meshrecovery, pipeline
// main.3 of 2026-11-20) fetches a workspace narrowed to that package's path (ADR 0499 clause 4; design 47 H, H-A-136:
// nothing else is fetched). The other packages' paths are not in it: they are no module of this build, never the
// module's directory, which holds no lock ("the package in . has no package-lock.json").
func TestANarrowedFetchOfOneFolderPackageLeavesTheOthersOut(t *testing.T) {
m, f := fetchMachine(t, folderFiles(""), nil)
args := map[string]any{"repository": "novox/mesh-contracts", "commit": commitSHA, "dir": ".", "paths": []any{"ts/seats/a"}}
out, err := m.fetch(context.Background(), args)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if strings.Join(r.Modules, ",") != "ts/seats/a" || strings.Join(r.Installed, ",") != "ts/seats/a" {
t.Errorf("modules %v, installed %v; want only the named package ts/seats/a", r.Modules, r.Installed)
}
if n := len(npmCalls(f)); n != 1 {
t.Errorf("%d npm commands, want the one install of the named package", n)
}
}
// The lint and test levels of that narrowed workspace run in the named package's path alone: a package whose path was
// left out is no target, and the module's directory is not tested whole in its place (H-A-136).
func TestANarrowedWorkspaceTestsOnlyItsPackage(t *testing.T) {
work := t.TempDir()
files := folderFiles("")
for p := range files {
if strings.HasPrefix(p, "ts/seats/b/") {
delete(files, p)
}
}
for p, body := range files {
write(t, filepath.Join(work, filepath.FromSlash(p)), body, 0o644)
}
for _, tc := range []struct {
narrowed bool
want string
}{{true, "ts/seats/a"}, {false, "."}} {
got, err := testTargets(work, ".", tc.narrowed)
if err != nil {
t.Fatal(err)
}
var where []string
for _, g := range got {
where = append(where, g.where)
}
if strings.Join(where, ",") != tc.want {
t.Errorf("narrowed %v: targets %v, want %s", tc.narrowed, where, tc.want)
}
}
}
@@ -43,9 +43,10 @@ type StepsAnswer struct {
Error string `json:"error,omitempty"`
}
// detectSteps finds the steps of the package directories under dir.
func detectSteps(dir string) ([]DetectedStep, error) {
targets, err := testTargets(dir, ".")
// detectSteps finds the steps of the package directories under dir; narrowedTree says the tree was narrowed to an
// artifact's paths, as testTargets reads it.
func detectSteps(dir string, narrowedTree bool) ([]DetectedStep, error) {
targets, err := testTargets(dir, ".", narrowedTree)
if err != nil {
return nil, err
}
@@ -176,7 +177,8 @@ func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]D
return nil, err
}
if len(req.Paths) > 0 {
if err := steprun.Narrow(tree, req); err != nil {
// Narrowed as the fetch of the step's workspace narrows (narrowTree), so the steps answered are those the levels run.
if err := narrowTree(tree, req); err != nil {
return nil, err
}
}
@@ -184,5 +186,5 @@ func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]D
if req.Dir != "" && req.Dir != "." {
dir = filepath.Join(tree, filepath.FromSlash(req.Dir))
}
return detectSteps(dir)
return detectSteps(dir, len(req.Paths) > 0)
}
@@ -22,17 +22,28 @@ func TestTheStepsOfATypeScriptArtifactAreFoundByItsScripts(t *testing.T) {
}
}
for name, c := range map[string]struct {
pkg string
want []DetectedStep
pkg string
narrowed bool
want []DetectedStep
}{
"both levels": {`{"scripts":{"test":"node t.js","test:integration":"node i.js"}}`,
"both levels": {`{"scripts":{"test":"node t.js","test:integration":"node i.js"}}`, false,
[]DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}, {Step: "integration", Packages: []string{"."}}}},
"unit only": {`{"scripts":{"test":"node t.js"}}`, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}}},
"no script": {`{"scripts":{}}`, []DetectedStep{{Step: "lint"}}},
"unit only": {`{"scripts":{"test":"node t.js"}}`, false, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}}},
"no script": {`{"scripts":{}}`, false, []DetectedStep{{Step: "lint"}}},
// A folder module's tree narrowed to ts/seats/a (H-A-136): the steps are those the narrowed levels run there.
"narrowed folder": {"", true, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"ts/seats/a"}}}},
} {
dir := t.TempDir()
put(dir, "package.json", c.pkg)
got, err := detectSteps(dir)
if c.narrowed {
for p, body := range folderFiles("") {
if !strings.HasPrefix(p, "ts/seats/b/") {
put(filepath.Join(dir, filepath.Dir(p)), filepath.Base(p), body)
}
}
} else {
put(dir, "package.json", c.pkg)
}
got, err := detectSteps(dir, c.narrowed)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s: steps = %+v (%v), want %+v", name, got, err, c.want)
}