Merge pull request 'mesh-build-node: narrow a step's workspace as H-A-136 and H-A-137 say; a narrowed-out package is no module of the step' (#716) from fix/47-H1-mesh-contracts-publish into main
This commit was merged in pull request #716.
This commit is contained in:
@@ -44,16 +44,37 @@ type artifact struct {
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// namedPath is the one path a package artifact names, cleaned, relative to its module's directory; false when it names
|
||||
// none, several, or one outside the module.
|
||||
func (a artifact) namedPath() (string, bool) {
|
||||
if len(a.Paths) != 1 {
|
||||
return "", false
|
||||
}
|
||||
p := path.Clean(strings.TrimSuffix(a.Paths[0], "/"))
|
||||
if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") {
|
||||
return "", false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
|
||||
// narrowedAway says whether the package's one named path is absent from the module's directory dir: in a workspace
|
||||
// narrowed to another artifact's paths, it is not this build's.
|
||||
func (a artifact) narrowedAway(dir string) bool {
|
||||
p, ok := a.namedPath()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := os.Lstat(filepath.Join(dir, filepath.FromSlash(p)))
|
||||
return os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// packageDir is the directory of a package artifact relative to its module's directory in dir: the one path it names
|
||||
// when that is a directory of the module holding a package.json (the contracts folders: ts/seats/<seat>/, each with its
|
||||
// own lock, ADR 0634 section 8), else the module's directory itself (a package that names no path, or source paths
|
||||
// beside the module's package.json: today's form).
|
||||
func (a artifact) packageDir(dir string) string {
|
||||
if len(a.Paths) != 1 {
|
||||
return "."
|
||||
}
|
||||
p := path.Clean(strings.TrimSuffix(a.Paths[0], "/"))
|
||||
if p == "." || path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") {
|
||||
p, ok := a.namedPath()
|
||||
if !ok {
|
||||
return "."
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, filepath.FromSlash(p), "package.json")); err != nil {
|
||||
|
||||
@@ -135,7 +135,7 @@ func (m *Module) testLevel(ctx context.Context, args map[string]any) (*TestResul
|
||||
if where == "" {
|
||||
where = "."
|
||||
}
|
||||
targets, err := testTargets(dir, where)
|
||||
targets, err := testTargets(dir, where, narrowed(filepath.Join(root, "tree")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -338,8 +338,9 @@ type testTarget struct{ dir, where string }
|
||||
|
||||
// testTargets are the directories a module's tests run in: each TypeScript package artifact's own path when every
|
||||
// package the module declares names one (ADR 0634 §8: its package.json, its lock, its tests), else the module's
|
||||
// directory.
|
||||
func testTargets(dir, where string) ([]testTarget, error) {
|
||||
// directory. In a workspace narrowed to one artifact's paths (narrowedTree), a package whose path was left out is no
|
||||
// target of this step (ADR 0499 clause 4; design 47 H, H-A-136).
|
||||
func testTargets(dir, where string, narrowedTree bool) ([]testTarget, error) {
|
||||
whole := []testTarget{{dir, where}}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
@@ -362,6 +363,9 @@ func testTargets(dir, where string) ([]testTarget, error) {
|
||||
if a.Kind != "package" {
|
||||
return whole, nil // a bundle is tested in its module's directory
|
||||
}
|
||||
if narrowedTree && a.narrowedAway(dir) {
|
||||
continue
|
||||
}
|
||||
at := a.packageDir(dir)
|
||||
if at == "." {
|
||||
return whole, nil
|
||||
|
||||
@@ -175,7 +175,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
|
||||
}
|
||||
}
|
||||
if len(req.Paths) > 0 {
|
||||
if err := steprun.Narrow(tree, req); err != nil {
|
||||
if err := narrowTree(tree, req); err != nil { // with the generated-from directories H-A-136 and H-A-137 keep
|
||||
return nil, err
|
||||
}
|
||||
// Said beside the tree, so the listing of the tree's modules leaves out those narrowed away.
|
||||
@@ -300,6 +300,11 @@ func (m *Module) typeScriptModules(ctx context.Context, tree string) (dirs []str
|
||||
continue
|
||||
}
|
||||
d := filepath.ToSlash(filepath.Dir(l))
|
||||
if a.Kind == "package" && narrowed(tree) && a.narrowedAway(filepath.Join(tree, filepath.FromSlash(d))) {
|
||||
// A package of the module whose own path the narrowing left out (ADR 0499 clause 4; design 47 H, H-A-136:
|
||||
// nothing else is fetched): no module of this build, never the module's directory, which holds no lock.
|
||||
continue
|
||||
}
|
||||
if a.Kind == "package" {
|
||||
d = path.Join(d, a.packageDir(filepath.Join(tree, filepath.FromSlash(d))))
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ func (m *Module) lintStep(ctx context.Context, args map[string]any) (any, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targets, err := testTargets(mdir, filepath.ToSlash(filepath.Clean(dir)))
|
||||
targets, err := testTargets(mdir, filepath.ToSlash(filepath.Clean(dir)), narrowed(filepath.Join(root, "tree")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
// A copy of mesh-build-go's narrowTree and generatedFromDirs (mesh-build-go/cmd/mesh-build-go/narrow.go) without its Go
|
||||
// part (the directories of the repository's packages a go.mod requires at the zero version). Copied, not moved to
|
||||
// mesh-delivery-sdk's steprun beside Narrow: the move is a change to two more repositories (mesh-delivery and back
|
||||
// into mesh-build-go) and a group delivery for some sixty lines, in a fix that unblocks the build modules' steps now;
|
||||
// the copy is said here so the move is not forgotten.
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// narrowedKeptAtRoot are what steprun.Narrow keeps beside the named paths, read from the request's directory: the
|
||||
// manifest and its lock files. A request narrowed from the repository's root keeps them at the root too, where the
|
||||
// asked module names none of them; they are removed again.
|
||||
var narrowedKeptAtRoot = []string{"module.json", "go.sum", "package-lock.json"}
|
||||
|
||||
// narrowTree narrows the fetched tree for a step of an artifact that names its paths (ADR 0499 clause 4,
|
||||
// steprun.Narrow). Beside the artifact's paths and the module's manifest, the workspace holds every generated-from
|
||||
// directory an artifact of the module names, so the unit level's generator check (steprun.OutOfStep) finds the seat
|
||||
// definitions and every generated package; nothing else (design 47 H, lead answers H-A-136 and H-A-137).
|
||||
func narrowTree(tree string, req sdkstep.FetchRequest) error {
|
||||
dir := cleanDir(req.Dir)
|
||||
if dir == "" {
|
||||
return steprun.Narrow(tree, req) // refused there: named paths without the module's directory
|
||||
}
|
||||
var outside []string
|
||||
inside := slices.Clone(req.Paths)
|
||||
for _, e := range generatedFromDirs(tree, dir) {
|
||||
if rel, ok := under(dir, e); ok {
|
||||
if !slices.Contains(inside, rel) {
|
||||
inside = append(inside, rel)
|
||||
}
|
||||
} else if !slices.Contains(outside, e) {
|
||||
outside = append(outside, e)
|
||||
}
|
||||
}
|
||||
if len(outside) == 0 {
|
||||
r := req
|
||||
r.Paths = inside
|
||||
return steprun.Narrow(tree, r)
|
||||
}
|
||||
// A directory outside the module's own: the request is narrowed from the root, with every path the module's
|
||||
// narrowing keeps named by its path from the root (the manifest and lock files only where the module has them).
|
||||
all := []string{}
|
||||
for _, p := range inside {
|
||||
all = append(all, path.Join(dir, p))
|
||||
}
|
||||
for _, f := range narrowedKeptAtRoot {
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(path.Join(dir, f)))); err == nil {
|
||||
all = append(all, path.Join(dir, f))
|
||||
}
|
||||
}
|
||||
all = append(all, outside...)
|
||||
if err := steprun.Narrow(tree, sdkstep.FetchRequest{Repository: req.Repository, Commit: req.Commit, Paths: all}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range narrowedKeptAtRoot {
|
||||
if !slices.Contains(all, f) {
|
||||
if err := os.RemoveAll(filepath.Join(tree, f)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanDir(dir string) string {
|
||||
if strings.TrimSpace(dir) == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/")
|
||||
}
|
||||
|
||||
// under answers p's path relative to dir when p is dir or lies under it.
|
||||
func under(dir, p string) (string, bool) {
|
||||
if dir == "." {
|
||||
return p, true
|
||||
}
|
||||
if p == dir {
|
||||
return ".", true
|
||||
}
|
||||
if rel, ok := strings.CutPrefix(p, dir+"/"); ok {
|
||||
return rel, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// generatedFromDirs are the generated-from directories (from the tree's root) the artifacts of the module in dir name
|
||||
// (H-A-136). A module at the repository's root keeps not its generated-from directory, which would be the whole
|
||||
// repository, but the generator's inputs and outputs: the seats/ directory of each generated-from directory and the
|
||||
// paths of every artifact of the module naming generated-from (H-A-137). A manifest that is absent or unreadable names
|
||||
// none; the steps that read it say so.
|
||||
func generatedFromDirs(tree, dir string) []string {
|
||||
raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(dir), "module.json"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
man, err := catalogue.Decode(raw)
|
||||
if err != nil || man.Build == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
add := func(p string) {
|
||||
if p == ".." || strings.HasPrefix(p, "../") || slices.Contains(out, p) {
|
||||
return
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(p))); err == nil { // steprun.Narrow refuses a path the tree lacks
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
for _, a := range man.Build.Artifacts {
|
||||
g := strings.TrimSpace(a.GeneratedFrom)
|
||||
if g == "" {
|
||||
continue
|
||||
}
|
||||
if dir != "." {
|
||||
add(path.Join(dir, filepath.ToSlash(g)))
|
||||
continue
|
||||
}
|
||||
add(path.Join(filepath.ToSlash(g), "seats"))
|
||||
for _, p := range a.Paths {
|
||||
add(path.Clean(filepath.ToSlash(p)))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatgen"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
)
|
||||
|
||||
// A root contracts module (mesh-contracts at 87eab81, pipeline main.3 of 2026-11-20): two seats, each generated as a Go
|
||||
// and a TypeScript package naming generated-from ".", in step with the seat generator, beside what no step reads.
|
||||
func rootContracts(t *testing.T) string {
|
||||
t.Helper()
|
||||
tree := t.TempDir()
|
||||
put := func(p, body string) {
|
||||
write(t, filepath.Join(tree, filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
var arts []string
|
||||
for _, s := range []string{"one-store", "two-store"} {
|
||||
put("seats/"+s+"/seat.json", `{"format":1,"seat":"`+s+`","scope":"mesh","grain":"whole","domain":"Module","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"},"answer":{"type":"object"}}]}`)
|
||||
pkg := strings.ReplaceAll(s, "-", "")
|
||||
arts = append(arts,
|
||||
`{"name":"go-`+pkg+`","kind":"package","language":"go","paths":["go/seats/`+pkg+`"],"generated-from":"."}`,
|
||||
`{"name":"ts-`+pkg+`","kind":"package","language":"typescript","paths":["ts/seats/`+pkg+`"],"generated-from":"."}`)
|
||||
}
|
||||
put("module.json", `{"module":"c","version":"0.1","build":{"artifacts":[`+strings.Join(arts, ",")+`]}}`)
|
||||
put("README.md", "the repository\n")
|
||||
g, err := seatgen.Load(tree, "example.org/c/go", "@example/c")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, err := seatgen.Generate(g)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for p, body := range files {
|
||||
put(p, string(body))
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-136 and H-A-137: a step's workspace of one package of a root module keeps, beside the package's path
|
||||
// and module.json, the seats/ directory and the paths of every artifact naming generated-from, so the unit level's
|
||||
// generator check finds nothing out of step; nothing else is kept. Narrowed bare (steprun.Narrow), the check names the
|
||||
// generated files left out.
|
||||
func TestANarrowedRootContractsModuleIsInStepWithTheGenerator(t *testing.T) {
|
||||
req := sdkstep.FetchRequest{Repository: "novox/c", Commit: commitSHA, Dir: ".", Paths: []string{"ts/seats/onestore"}}
|
||||
bare := rootContracts(t)
|
||||
if err := steprun.Narrow(bare, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := steprun.OutOfStep(bare, steprun.GeneratedRootTS); err != nil || len(out) == 0 {
|
||||
t.Fatalf("narrowed bare: out of step %v (%v), want the left-out packages named", out, err)
|
||||
}
|
||||
tree := rootContracts(t)
|
||||
if err := narrowTree(tree, req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out, err := steprun.OutOfStep(tree, steprun.GeneratedRootTS); err != nil || len(out) != 0 {
|
||||
t.Errorf("narrowed: out of step %v (%v), want none", out, err)
|
||||
}
|
||||
for _, kept := range []string{"module.json", "seats/two-store/seat.json", "ts/seats/twostore/package.json", "go/seats/twostore/go.mod"} {
|
||||
if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(kept))); err != nil {
|
||||
t.Errorf("%s is not kept: %v", kept, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(tree, "README.md")); err == nil {
|
||||
t.Error("README.md is still in the workspace")
|
||||
}
|
||||
}
|
||||
@@ -91,3 +91,54 @@ func TestAFolderModulesPackageIsBuiltAndTestedInItsPath(t *testing.T) {
|
||||
t.Errorf("unit level = %+v, ran tests in %v, want both package paths", r, tests)
|
||||
}
|
||||
}
|
||||
|
||||
// A step of one package of a root module with several (mesh-contracts' ts-meshbroker beside ts-meshrecovery, pipeline
|
||||
// main.3 of 2026-11-20) fetches a workspace narrowed to that package's path (ADR 0499 clause 4; design 47 H, H-A-136:
|
||||
// nothing else is fetched). The other packages' paths are not in it: they are no module of this build, never the
|
||||
// module's directory, which holds no lock ("the package in . has no package-lock.json").
|
||||
func TestANarrowedFetchOfOneFolderPackageLeavesTheOthersOut(t *testing.T) {
|
||||
m, f := fetchMachine(t, folderFiles(""), nil)
|
||||
args := map[string]any{"repository": "novox/mesh-contracts", "commit": commitSHA, "dir": ".", "paths": []any{"ts/seats/a"}}
|
||||
out, err := m.fetch(context.Background(), args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := out.(FetchResult)
|
||||
if strings.Join(r.Modules, ",") != "ts/seats/a" || strings.Join(r.Installed, ",") != "ts/seats/a" {
|
||||
t.Errorf("modules %v, installed %v; want only the named package ts/seats/a", r.Modules, r.Installed)
|
||||
}
|
||||
if n := len(npmCalls(f)); n != 1 {
|
||||
t.Errorf("%d npm commands, want the one install of the named package", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The lint and test levels of that narrowed workspace run in the named package's path alone: a package whose path was
|
||||
// left out is no target, and the module's directory is not tested whole in its place (H-A-136).
|
||||
func TestANarrowedWorkspaceTestsOnlyItsPackage(t *testing.T) {
|
||||
work := t.TempDir()
|
||||
files := folderFiles("")
|
||||
for p := range files {
|
||||
if strings.HasPrefix(p, "ts/seats/b/") {
|
||||
delete(files, p)
|
||||
}
|
||||
}
|
||||
for p, body := range files {
|
||||
write(t, filepath.Join(work, filepath.FromSlash(p)), body, 0o644)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
narrowed bool
|
||||
want string
|
||||
}{{true, "ts/seats/a"}, {false, "."}} {
|
||||
got, err := testTargets(work, ".", tc.narrowed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var where []string
|
||||
for _, g := range got {
|
||||
where = append(where, g.where)
|
||||
}
|
||||
if strings.Join(where, ",") != tc.want {
|
||||
t.Errorf("narrowed %v: targets %v, want %s", tc.narrowed, where, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,9 +43,10 @@ type StepsAnswer struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// detectSteps finds the steps of the package directories under dir.
|
||||
func detectSteps(dir string) ([]DetectedStep, error) {
|
||||
targets, err := testTargets(dir, ".")
|
||||
// detectSteps finds the steps of the package directories under dir; narrowedTree says the tree was narrowed to an
|
||||
// artifact's paths, as testTargets reads it.
|
||||
func detectSteps(dir string, narrowedTree bool) ([]DetectedStep, error) {
|
||||
targets, err := testTargets(dir, ".", narrowedTree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -176,7 +177,8 @@ func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]D
|
||||
return nil, err
|
||||
}
|
||||
if len(req.Paths) > 0 {
|
||||
if err := steprun.Narrow(tree, req); err != nil {
|
||||
// Narrowed as the fetch of the step's workspace narrows (narrowTree), so the steps answered are those the levels run.
|
||||
if err := narrowTree(tree, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -184,5 +186,5 @@ func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]D
|
||||
if req.Dir != "" && req.Dir != "." {
|
||||
dir = filepath.Join(tree, filepath.FromSlash(req.Dir))
|
||||
}
|
||||
return detectSteps(dir)
|
||||
return detectSteps(dir, len(req.Paths) > 0)
|
||||
}
|
||||
|
||||
@@ -22,17 +22,28 @@ func TestTheStepsOfATypeScriptArtifactAreFoundByItsScripts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
pkg string
|
||||
want []DetectedStep
|
||||
pkg string
|
||||
narrowed bool
|
||||
want []DetectedStep
|
||||
}{
|
||||
"both levels": {`{"scripts":{"test":"node t.js","test:integration":"node i.js"}}`,
|
||||
"both levels": {`{"scripts":{"test":"node t.js","test:integration":"node i.js"}}`, false,
|
||||
[]DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}, {Step: "integration", Packages: []string{"."}}}},
|
||||
"unit only": {`{"scripts":{"test":"node t.js"}}`, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}}},
|
||||
"no script": {`{"scripts":{}}`, []DetectedStep{{Step: "lint"}}},
|
||||
"unit only": {`{"scripts":{"test":"node t.js"}}`, false, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}}},
|
||||
"no script": {`{"scripts":{}}`, false, []DetectedStep{{Step: "lint"}}},
|
||||
// A folder module's tree narrowed to ts/seats/a (H-A-136): the steps are those the narrowed levels run there.
|
||||
"narrowed folder": {"", true, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"ts/seats/a"}}}},
|
||||
} {
|
||||
dir := t.TempDir()
|
||||
put(dir, "package.json", c.pkg)
|
||||
got, err := detectSteps(dir)
|
||||
if c.narrowed {
|
||||
for p, body := range folderFiles("") {
|
||||
if !strings.HasPrefix(p, "ts/seats/b/") {
|
||||
put(filepath.Join(dir, filepath.Dir(p)), filepath.Base(p), body)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
put(dir, "package.json", c.pkg)
|
||||
}
|
||||
got, err := detectSteps(dir, c.narrowed)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: steps = %+v (%v), want %+v", name, got, err, c.want)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user