From d5804927b303b61cb94b8de2c7d3ba6639410c6d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 21:04:59 +0200 Subject: [PATCH] Make the build modules' fetch merge the head onto a trunk commit and place siblings beside the tree, so the checker judges the merge and tests read their siblings (issues 1675, 1676, 1677) --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 47 ++- .../cmd/mesh-build-go/fetch_test.go | 8 +- .../cmd/mesh-build-go/integration.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/module.go | 4 + .../cmd/mesh-build-go/preview.go | 173 +++++++++++ .../mesh-build-go/preview_integration_test.go | 228 +++++++++++++++ .../cmd/mesh-build-go/preview_test.go | 270 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/test.go | 69 ++++- .../testdata/repos/beside/beside.go | 1 + .../testdata/repos/beside/beside_test.go | 22 ++ .../testdata/repos/beside/go.mod | 3 + modules/mesh-build-go/go.mod | 2 +- modules/mesh-build-go/go.sum | 2 + .../cmd/mesh-build-node/fetch.go | 30 +- .../cmd/mesh-build-node/harness_test.go | 2 +- .../cmd/mesh-build-node/module.go | 10 +- .../cmd/mesh-build-node/preview.go | 93 ++++++ .../preview_integration_test.go | 95 ++++++ .../cmd/mesh-build-node/preview_test.go | 87 ++++++ modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 2 + 21 files changed, 1134 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/preview.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/preview_integration_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/preview_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside_test.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/go.mod create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/preview.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/preview_integration_test.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/preview_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index bddf11ac..dd4fd719 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -17,6 +17,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/buildstep" ) +var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + // ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the // remote of a fetch is ssh:////.git. const ForgeAlias = "mesh-forge" @@ -37,6 +39,18 @@ type FetchResult struct { Workspace string `json:"workspace"` Commit string `json:"commit"` Modules []string `json:"modules"` + // The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the + // head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a + // module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the + // files, and the workspace is left at the trunk commit. + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + Trees map[string]ModuleTree `json:"trees,omitempty"` + // Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which + // build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4). + Beside []Sibling `json:"beside,omitempty"` } // fetchTool is the fetch step tool. @@ -52,6 +66,12 @@ func (m *Module) fetchTool() StepTool { "repository": map[string]any{"type": "string", "description": "/ on the forge"}, "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, "ref": map[string]any{"type": "string", "description": "refs/pull//head when only a pull request's head carries the commit"}, + "base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"}, + "beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/, which build_go_test gives the tests as MESH_CHECK_BESIDE", + "items": map[string]any{"type": "object", "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/ on the forge"}, + "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, + }, "required": []string{"repository", "commit"}}}, }, "required": []string{"repository", "commit"}, }, @@ -64,6 +84,14 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { repository, _ := args["repository"].(string) commit, _ := args["commit"].(string) ref, _ := args["ref"].(string) + base, _ := args["base"].(string) + if base != "" && !commitOK.MatchString(base) { + return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base) + } + siblings, err := siblingsArg(args["beside"]) + if err != nil { + return nil, err + } if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { return nil, fmt.Errorf("%q is not a repository, /", repository) } @@ -74,14 +102,27 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { if err := os.MkdirAll(tree, 0o700); err != nil { return nil, err } - remote := "ssh://" + ForgeAlias + "/" + repository + ".git" - if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil { + remote := m.remote(repository) + if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil { + return nil, err + } + res := FetchResult{Workspace: id, Commit: commit} + if base != "" { + if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil { + return nil, err + } + } + if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil { return nil, err } dirs, err := m.goModDirs(ctx, tree) if err != nil { return nil, err } + res.Modules = dirs + if base != "" && res.Merge == "" { + return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested + } for _, d := range dirs { if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil { continue // a module that vendors resolves nothing @@ -93,7 +134,7 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { } } } - return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil + return res, nil } // goModDirs are the directories of the tree that hold a go.mod, relative to it, in order. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index d223cdaa..56657e06 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -30,7 +30,8 @@ type fakeStep struct { gomods string gate chan struct{} // when set, the first git command waits for it goRule func(line string) forgekey.Ran - vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout + vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout + rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do } func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { @@ -41,6 +42,11 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s if f.gate != nil && strings.Contains(line, " init ") { <-f.gate } + if f.rule != nil { + if r, ok := f.rule(line); ok { + return r + } + } switch { case name == "git" && strings.Contains(line, " rev-parse HEAD"): return forgekey.Ran{Stdout: commitSHA + "\n"} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index d96c4bb5..1f112ed8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -60,7 +60,7 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag return nil, fmt.Errorf("the integration level's bus: %w", err) } defer stopBus() - env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats} + env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...) // Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run // below finds them built. Whether they compile is for the run to say; its failure is not read here. for _, d := range dirs { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 8851a8ff..d0c4d603 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -37,6 +37,10 @@ type Module struct { // both outside the account's home (ADR 0466 §1); Home is the account's home. Run forgekey.Runner Home, Work, Cache string + // RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of + // the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote). + RemoteBase string + FetchOptions []buildstep.Option // HTTP fetches the files an archive artifact is pinned to (url and sha256); a client of its own with a fifteen-minute // timeout when nil. A test serves its own. HTTP *http.Client diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview.go b/modules/mesh-build-go/cmd/mesh-build-go/preview.go new file mode 100644 index 00000000..da279e44 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview.go @@ -0,0 +1,173 @@ +package main + +// The merge preview and the siblings of a fetch (novox/hq ADR 0484 clauses 3 and 4; issues 1675 and 1677). The merge +// itself is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes; what is here is the answer +// built from it and the placing of the sibling repositories the tests read. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// Sibling is a repository placed beside the fetched tree: its name (the directory under beside/) and its commit. +type Sibling struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Dir string `json:"dir"` +} + +// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it. +func (m *Module) remote(repository string) string { + base := m.RemoteBase + if base == "" { + base = "ssh://" + ForgeAlias + "/" + } + return base + repository + ".git" +} + +// merge makes the merge preview of the head onto base and writes the answer into res. +func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error { + mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...) + if err != nil { + return err + } + res.Base = mr.Base + res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if res.Merge == "" { + return nil + } + res.Trees, err = m.moduleTrees(ctx, tree, res.Merge) + return err +} + +// ModuleTree is what the merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256 +// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs. +type ModuleTree struct { + Tree string `json:"tree"` + Manifest string `json:"manifest"` + Locks map[string]string `json:"locks,omitempty"` +} + +var lockFiles = []string{"go.sum", "package-lock.json"} + +// moduleTrees are the trees, in the merge commit, of every directory of the tree that holds a module.json; the workspace +// is at the merge, so the manifest and the locks are read from it. +func (m *Module) moduleTrees(ctx context.Context, tree, merge string) (map[string]ModuleTree, error) { + r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + trees := map[string]ModuleTree{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l == "" { + continue + } + dir := filepath.ToSlash(filepath.Dir(l)) + spec := merge + ":" + dir + if dir == "." { + spec = merge + "^{tree}" + } + t := m.Run(ctx, nil, "git", "-C", tree, "rev-parse", spec) + if t.Status != 0 || t.Err != "" { + return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr)) + } + mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)} + raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l))) + if err != nil { + return nil, err + } + mt.Manifest = sha256Hex(raw) + for _, lock := range lockFiles { + if raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(dir), lock)); err == nil { + if mt.Locks == nil { + mt.Locks = map[string]string{} + } + mt.Locks[lock] = sha256Hex(raw) + } + } + trees[dir] = mt + } + return trees, nil +} + +func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } + +type siblingArg struct{ repository, commit string } + +func siblingsArg(v any) ([]siblingArg, error) { + if v == nil { + return nil, nil + } + list, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("beside is a list of {repository, commit}") + } + var out []siblingArg + seen := map[string]bool{} + for _, e := range list { + o, _ := e.(map[string]any) + repository, _ := o["repository"].(string) + commit, _ := o["commit"].(string) + if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { + return nil, fmt.Errorf("beside: %q is not a repository, /", repository) + } + if !commitOK.MatchString(commit) { + return nil, fmt.Errorf("beside: %q is not a full commit (40 hex digits)", commit) + } + name := repository[strings.Index(repository, "/")+1:] + if seen[name] { + return nil, fmt.Errorf("beside: two repositories are named %s", name) + } + seen[name] = true + out = append(out, siblingArg{repository, commit}) + } + return out, nil +} + +// BesideDir is where a workspace's siblings are placed, and what MESH_CHECK_BESIDE names. +func (m *Module) besideDir(workspace string) string { + return filepath.Join(m.Work, workspace, "beside") +} + +// placeSiblings fetches each sibling into beside/ of the workspace root, as a plain directory. A symlink anywhere +// in a placed tree fails the step naming it: the mesh creates none, and a test must not be led out of its sibling. +func (m *Module) placeSiblings(ctx context.Context, root string, siblings []siblingArg) ([]Sibling, error) { + var placed []Sibling + for _, s := range siblings { + name := s.repository[strings.Index(s.repository, "/")+1:] + dir := filepath.Join(root, "beside", name) + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, "", m.FetchOptions...); err != nil { + return nil, fmt.Errorf("beside %s: %w", s.repository, err) + } + if link := firstSymlink(dir); link != "" { + return nil, fmt.Errorf("beside %s holds a symlink, %s", s.repository, link) + } + placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name}) + } + sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir }) + return placed, nil +} + +func firstSymlink(dir string) string { + var found string + filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.Type()&fs.ModeSymlink != 0 && found == "" { + found, _ = filepath.Rel(dir, p) + return fs.SkipAll + } + return nil + }) + return found +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/preview_integration_test.go new file mode 100644 index 00000000..a800d22d --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview_integration_test.go @@ -0,0 +1,228 @@ +//go:build integration + +package main + +// The fetch step's merge and siblings on real git repositories and the real go (issues 1675 and 1677): they start +// processes, so they carry the tag; preview_test.go proves the commands with a fake runner. + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func git(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} + +func put(t *testing.T, dir, name, body string) { + t.Helper() + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +// forge is a directory of bare repositories /.git standing in for the forge, and a module fetching from it. +type forge struct { + t *testing.T + root string +} + +func newForge(t *testing.T) *forge { return &forge{t: t, root: t.TempDir()} } + +// repo makes the repository owner/name with the files on main and answers its work directory and the commit. +func (f *forge) repo(name string, files map[string]string) (work, commit string) { + bare := filepath.Join(f.root, "forge", name+".git") + work = filepath.Join(f.root, "work", name) + os.MkdirAll(bare, 0o755) + os.MkdirAll(work, 0o755) + git(f.t, bare, "init", "-q", "--bare") + git(f.t, work, "init", "-q", "-b", "main") + for n, c := range files { + put(f.t, work, n, c) + } + git(f.t, work, "add", ".") + git(f.t, work, "commit", "-q", "-m", "one") + git(f.t, work, "push", "-q", bare, "HEAD:refs/heads/main") + return work, git(f.t, work, "rev-parse", "HEAD") +} + +// commit makes a commit off `from` in the repository's work directory and publishes it as the pull request's head. +func (f *forge) commit(name, work, from string, files map[string]string, ref string) string { + git(f.t, work, "checkout", "-q", "-B", "b"+ref[len(ref)-3:], from) + for n, c := range files { + put(f.t, work, n, c) + } + git(f.t, work, "add", ".") + git(f.t, work, "commit", "-q", "-m", "change") + sha := git(f.t, work, "rev-parse", "HEAD") + git(f.t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", name+".git"), "HEAD:"+ref) + return sha +} + +func (f *forge) module(t *testing.T) (*Module, *keptStates) { + m, states := fetchModule(t, &fakeStep{}) + m.Run = StepRunner(m.Home) + m.RemoteBase = "file://" + filepath.Join(f.root, "forge") + "/" + m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()} + return m, states +} + +// waitLong is waitEnd for a step that runs the real go, whose first build in a cold cache takes longer than 3 s. +func waitLong(t *testing.T, s *keptStates) delivery.Step { + t.Helper() + select { + case key := <-s.ended: + s.mu.Lock() + defer s.mu.Unlock() + v := s.steps[key] + return v[len(v)-1] + case <-time.After(2 * time.Minute): + t.Fatal("the step never ended on its key") + return delivery.Step{} + } +} + +func fetched(t *testing.T, m *Module, states *keptStates, args map[string]any) (FetchResult, delivery.Step) { + t.Helper() + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res FetchResult + if st.State == delivery.StepPassed { + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + } + return res, st +} + +var baseFiles = map[string]string{ + "go.mod": "module fixture/m\n\ngo 1.26\n", + "a.txt": "one\ntwo\nthree\n", + "modules/m/module.json": "{\"name\":\"m\"}\n", + "modules/m/go.sum": "", + "modules/m/m.txt": "m\n", + "modules/other/other.txt": "o\n", +} + +// 1675/AC1 and AC2 on real git: a clean merge answers the merge commit, the changed paths (the trunk's change is not +// among them) and the tree of each module directory, which equals git's own; a conflict is a passed step naming the files, +// leaving the workspace at the trunk commit; and no base is today's fetch. +func TestRealFetchMergesConflictsAndPlainlyFetches(t *testing.T) { + f := newForge(t) + work, fork := f.repo("o/r", baseFiles) + trunk := f.commit("o/r", work, fork, map[string]string{"modules/other/other.txt": "trunk\n", "a.txt": "one\ntwo\nTRUNK\n"}, "refs/heads/main") + clean := f.commit("o/r", work, fork, map[string]string{"modules/m/m.txt": "head\n"}, "refs/pull/1/head") + clash := f.commit("o/r", work, fork, map[string]string{"a.txt": "one\ntwo\nHEAD\n"}, "refs/pull/2/head") + m, states := f.module(t) + + res, st := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk}) + if st.State != delivery.StepPassed || res.Merge == "" || res.Base != trunk || res.Commit != clean || len(res.Conflicts) != 0 { + t.Fatalf("clean: %s %s", st.State, st.Result) + } + if strings.Join(res.Changed, ",") != "modules/m/m.txt" { + t.Errorf("changed = %v, want the head's own change alone", res.Changed) + } + tree := res.Trees["modules/m"] + if want := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", res.Merge+":modules/m"); tree.Tree != want { + t.Errorf("tree = %s, want %s", tree.Tree, want) + } + if tree.Manifest != sha256Hex([]byte("{\"name\":\"m\"}\n")) || tree.Locks["go.sum"] != sha256Hex(nil) { + t.Errorf("manifest and locks = %+v", tree) + } + if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != res.Merge { + t.Errorf("the workspace is at %s, want the merge %s", got, res.Merge) + } + + res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk}) + if st.State != delivery.StepPassed || res.Merge != "" || strings.Join(res.Conflicts, ",") != "a.txt" { + t.Fatalf("conflict: %s %s", st.State, st.Result) + } + if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk { + t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk) + } + + res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"}) + if st.State != delivery.StepPassed || res.Merge != "" || res.Base != "" || res.Trees != nil || len(res.Modules) != 1 { + t.Errorf("no base: %s %s", st.State, st.Result) + } +} + +// 1677/AC1 on the real go: the unit level of a repository whose test reads MESH_CHECK_BESIDE fails loudly without the +// sibling and passes with it; the sibling is a plain directory with no symlink in it; and a sibling that holds a +// symlink fails the fetch naming it. +func TestRealTestsReadTheSiblingBesideTheTree(t *testing.T) { + f := newForge(t) + files := map[string]string{} + for _, n := range []string{"go.mod", "beside.go", "beside_test.go"} { + raw, err := os.ReadFile(filepath.Join("testdata", "repos", "beside", n)) + if err != nil { + t.Fatal(err) + } + files[n] = string(raw) + } + _, head := f.repo("o/r", files) + _, sibling := f.repo("o/sibling", map[string]string{"data.txt": "from the sibling\n"}) + m, states := f.module(t) + + without, _ := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head}) + if _, err := m.testToolRun(t, without.Workspace); err != nil { + t.Fatal(err) + } + var res TestResult + st := waitLong(t, states) + json.Unmarshal(st.Result, &res) + if st.State != delivery.StepPassed || res.Passed || len(res.Failed) != 1 || !strings.Contains(res.Failed[0].Output, "MESH_CHECK_BESIDE is not set") { + t.Fatalf("without the sibling: %s %s", st.State, st.Result) + } + + with, st2 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head, + "beside": []any{map[string]any{"repository": "o/sibling", "commit": sibling}}}) + if st2.State != delivery.StepPassed || len(with.Beside) != 1 { + t.Fatalf("with the sibling: %s %s", st2.State, st2.Result) + } + m.testToolRun(t, with.Workspace) + res = TestResult{} + st = waitLong(t, states) + json.Unmarshal(st.Result, &res) + if !res.Passed { + t.Errorf("with the sibling the level failed: %s", st.Result) + } + if link := firstSymlink(filepath.Join(m.Work, with.Workspace, "beside")); link != "" { + t.Errorf("the placed tree holds a symlink, %s", link) + } + + _, bad := f.repo("o/linked", map[string]string{"data.txt": "x\n"}) + work := filepath.Join(f.root, "work", "o/linked") + if err := os.Symlink("/etc", filepath.Join(work, "etc-link")); err != nil { + t.Fatal(err) + } + git(t, work, "add", ".") + git(t, work, "commit", "-q", "-m", "link") + bad = git(t, work, "rev-parse", "HEAD") + git(t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", "o/linked.git"), "HEAD:refs/heads/main") + _, st3 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head, + "beside": []any{map[string]any{"repository": "o/linked", "commit": bad}}}) + if st3.State != delivery.StepFailed || !strings.Contains(st3.Reason, "etc-link") { + t.Errorf("a sibling with a symlink: %s %s", st3.State, st3.Reason) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go new file mode 100644 index 00000000..4a1e7614 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/preview_test.go @@ -0,0 +1,270 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const ( + baseSHA = "abcdef0123456789abcdef0123456789abcdef01" + mergeSHA = "fedcba9876543210fedcba9876543210fedcba98" + treeSHA = "1111111111111111111111111111111111111111" +) + +// previewRules answers the merge's commands as git does: the merge commit, the changed paths, the manifests, the trees. +func previewRules(conflict bool) func(string) (forgekey.Ran, bool) { + merged := false + return func(line string) (forgekey.Ran, bool) { + switch { + case strings.Contains(line, "--is-shallow-repository"): + return forgekey.Ran{Stdout: "true\n"}, true + case strings.Contains(line, " merge --no-ff"): + if conflict { + return forgekey.Ran{Status: 1, Stderr: "CONFLICT (content): Merge conflict in a.go\n"}, true + } + merged = true + return forgekey.Ran{}, true + case strings.Contains(line, "diff --name-only --diff-filter=U"): + return forgekey.Ran{Stdout: "a.go\nsub/b.go\n"}, true + case strings.Contains(line, "diff --name-only --no-renames"): + return forgekey.Ran{Stdout: "a.go\nmodules/m/main.go\n"}, true + case strings.Contains(line, " rev-parse HEAD") && merged: + return forgekey.Ran{Stdout: mergeSHA + "\n"}, true + case strings.Contains(line, " ls-files ") && strings.Contains(line, "module.json"): + return forgekey.Ran{Stdout: "modules/m/module.json\n"}, true + case strings.Contains(line, " rev-parse "+mergeSHA+":modules/m"): + return forgekey.Ran{Stdout: treeSHA + "\n"}, true + } + return forgekey.Ran{}, false + } +} + +// 1675/AC1: with a base the fetch merges the head onto it with the SDK's merge (the identity and flags of the check +// merge, hooks off), and answers base, merge commit, changed paths and, per module directory, tree, manifest hash and lock hashes. +func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n", rule: previewRules(false)} + m, states := fetchModule(t, f) + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + // the workspace's module.json and go.sum are on disk once the fake checks the head out + for i, a := range args { + if a == "-C" && len(args) > i+1 && strings.Contains(strings.Join(args, " "), " checkout ") { + os.MkdirAll(filepath.Join(args[i+1], "modules", "m"), 0o755) + os.WriteFile(filepath.Join(args[i+1], "modules", "m", "module.json"), []byte("{}\n"), 0o644) + os.WriteFile(filepath.Join(args[i+1], "modules", "m", "go.sum"), []byte("sum\n"), 0o644) + } + } + return f.run(ctx, env, name, args...) + } + args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA} + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + var res FetchResult + if err := json.Unmarshal(st.Result, &res); err != nil { + t.Fatal(err) + } + if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 || + strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" { + t.Errorf("result = %s", st.Result) + } + mt, ok := res.Trees["modules/m"] + if !ok || mt.Tree != treeSHA || mt.Manifest != sha256Hex([]byte("{}\n")) || mt.Locks["go.sum"] != sha256Hex([]byte("sum\n")) { + t.Errorf("trees = %+v", res.Trees) + } + lines := strings.Join(f.lines(), "\n") + for _, want := range []string{"fetch --no-tags --unshallow origin " + baseSHA, "checkout -q --detach " + baseSHA, + "core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} { + if !strings.Contains(lines, want) { + t.Errorf("git was not asked %q:\n%s", want, lines) + } + } +} + +// 1675/AC2: a conflict is an answer: the step ends passed, the files are named, no merge commit, nothing downloaded. +func TestFetchWithAConflictEndsPassedAndNamesTheFiles(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n", rule: previewRules(true)} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA} + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != delivery.StepPassed { + t.Fatalf("a conflict ended the step %s: %s", st.State, st.Reason) + } + var res FetchResult + json.Unmarshal(st.Result, &res) + if res.Merge != "" || res.Base != baseSHA || strings.Join(res.Conflicts, ",") != "a.go,sub/b.go" || len(res.Changed) != 0 { + t.Errorf("result = %s", st.Result) + } + lines := strings.Join(f.lines(), "\n") + if !strings.Contains(lines, "merge --abort") || strings.Contains(lines, "mod download") { + t.Errorf("the merge was not undone, or modules were downloaded:\n%s", lines) + } +} + +// 1675/AC3: no base is today's fetch: no merge, no extra command, and the answer carries none of the preview. +func TestFetchWithoutABaseIsTodaysFetch(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + fetchTool(t, m)(fetchArgs) + st := states.waitEnd(t) + lines := strings.Join(f.lines(), "\n") + if strings.Contains(lines, "merge") || strings.Contains(lines, "unshallow") { + t.Errorf("a fetch without a base merged:\n%s", lines) + } + for _, key := range []string{"base", "merge", "conflicts", "changed", "trees", "beside"} { + if strings.Contains(string(st.Result), `"`+key+`"`) { + t.Errorf("the answer carries %q without a base: %s", key, st.Result) + } + } +} + +func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) { + f := &fakeStep{} + m, states := fetchModule(t, f) + fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": "main"}) + if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 { + t.Errorf("the step ended %s after %d commands", st.State, len(f.lines())) + } +} + +// 1677/AC1: each sibling is fetched into beside/ of the workspace, and the tests of the workspace are given +// MESH_CHECK_BESIDE naming that directory, at the unit level; a workspace without siblings is given none. +func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{ + map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}, + map[string]any{"repository": "novox/mesh-host", "commit": commitSHA}}} + if _, err := fetchTool(t, m)(args); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + var res FetchResult + json.Unmarshal(st.Result, &res) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + if len(res.Beside) != 2 || res.Beside[0].Dir != "mesh-catalog" || res.Beside[1].Dir != "mesh-host" { + t.Fatalf("result = %s", st.Result) + } + lines := strings.Join(f.lines(), "\n") + for _, d := range []string{"beside/mesh-catalog", "beside/mesh-host"} { + if !strings.Contains(lines, filepath.Join(m.Work, res.Workspace, d)+" init") && !strings.Contains(lines, "-C "+filepath.Join(m.Work, res.Workspace, d)) { + t.Errorf("nothing was fetched into %s:\n%s", d, lines) + } + } + if !strings.Contains(lines, "remote add origin ssh://mesh-forge/novox/mesh-catalog.git") { + t.Errorf("the sibling was not fetched over ssh:\n%s", lines) + } + + calls := &[]cmdRecord{} + m.Run = capturedRunner(t, "failing", calls) + if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := m.testToolRun(t, res.Workspace); err != nil { + t.Fatal(err) + } + states.waitEnd(t) + want := "MESH_CHECK_BESIDE=" + filepath.Join(m.Work, res.Workspace, "beside") + if !envHas(*calls, "go", want) { + t.Errorf("no go command was given %s", want) + } +} + +func (m *Module) testToolRun(t *testing.T, ws string) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_test" { + return tool.Run(map[string]any{"workspace": ws, "level": "unit"}) + } + } + t.Fatal("no tool build_go_test") + return nil, nil +} + +func envHas(calls []cmdRecord, name, want string) bool { + for _, c := range calls { + if strings.HasPrefix(c.line, name+" ") { + for _, e := range c.env { + if e == want { + return true + } + } + } + } + return false +} + +func TestAWorkspaceWithoutSiblingsIsGivenNoBesideVariable(t *testing.T) { + m, states, ws, calls := workspaceFor(t, "failing") + runUnit(t, m, states, ws) + for _, c := range *calls { + for _, e := range c.env { + if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") { + t.Errorf("%s was given %s", c.line, e) + } + } + } +} + +func TestSiblingsThatAreNotRepositoriesOrCommitsAreRefused(t *testing.T) { + for name, beside := range map[string]any{ + "not a list": "x", + "bad repository": []any{map[string]any{"repository": "../x/y", "commit": baseSHA}}, + "bad commit": []any{map[string]any{"repository": "novox/a", "commit": "main"}}, + "twice": []any{map[string]any{"repository": "novox/a", "commit": baseSHA}, map[string]any{"repository": "other/a", "commit": baseSHA}}, + } { + f := &fakeStep{} + m, states := fetchModule(t, f) + fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "beside": beside}) + if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 { + t.Errorf("%s: the step ended %s after %d commands", name, st.State, len(f.lines())) + } + } +} + +// 1288/AC5 (the checker's contract): an import path runs only in the go.mod directory whose module holds it; a +// relative pattern runs in every directory; no pattern is ./... everywhere. +func TestPackagesRunOnlyInTheModuleThatHoldsThem(t *testing.T) { + tree := t.TempDir() + for dir, mod := range map[string]string{".": "example.org/root", "tools": "example.org/root/tools", "other": "example.org/other"} { + os.MkdirAll(filepath.Join(tree, dir), 0o755) + os.WriteFile(filepath.Join(tree, dir, "go.mod"), []byte("module "+mod+"\n\ngo 1.26\n"), 0o644) + } + dirs := []string{".", "other", "tools"} + got, err := packagesByDir(tree, dirs, []string{"example.org/root/a", "example.org/root/tools/b", "example.org/other"}) + if err != nil { + t.Fatal(err) + } + want := map[string]string{".": "example.org/root/a", "tools": "example.org/root/tools/b", "other": "example.org/other"} + for d, w := range want { + if strings.Join(got[d], ",") != w { + t.Errorf("%s runs %v, want %s", d, got[d], w) + } + } + got, _ = packagesByDir(tree, dirs, []string{"./x/..."}) + if len(got["."]) != 1 || len(got["tools"]) != 1 || len(got["other"]) != 1 { + t.Errorf("a relative pattern = %v", got) + } + got, _ = packagesByDir(tree, dirs, nil) + if got["tools"][0] != "./..." { + t.Errorf("no pattern = %v", got) + } + if _, err := packagesByDir(tree, dirs, []string{"example.org/nowhere/x"}); err == nil || !strings.Contains(err.Error(), "example.org/nowhere/x") { + t.Errorf("a package no module holds: %v", err) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 426cdd44..3448fbd1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -65,13 +65,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { } switch level { case LevelUnit: - return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}}) + return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)}) case LevelIntegration: return m.integration(ctx, workspace, tree, packages) } return nil, fmt.Errorf("the level %q is neither unit nor integration", level) } +// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484 +// clause 4), when it placed any; the tests that read other repositories look there (mesh-sdk go/internal/beside). +func (m *Module) besideEnv(workspace string) []string { + dir := m.besideDir(workspace) + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + return nil + } + return []string{"MESH_CHECK_BESIDE=" + dir} +} + func packagesArg(v any) ([]string, error) { if v == nil { return nil, nil @@ -115,12 +125,16 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR return nil, err } res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus} - packages := o.packages - if len(packages) == 0 { - packages = []string{"./..."} + byDir, err := packagesByDir(tree, dirs, o.packages) + if err != nil { + return nil, err } began := m.clock() for _, d := range dirs { + packages := byDir[d] + if len(packages) == 0 { + continue // no package asked for lies in this module + } args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout} if len(o.tags) > 0 { args = append(args, "-tags", strings.Join(o.tags, ",")) @@ -141,3 +155,50 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR res.judge(o.bounds) return res, nil } + +// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as +// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it +// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern +// means ./... everywhere. An import path no module of the tree holds is an error that names it. +func packagesByDir(tree string, dirs, patterns []string) (map[string][]string, error) { + out := map[string][]string{} + if len(patterns) == 0 { + patterns = []string{"./..."} + } + modules := map[string]string{} + for _, d := range dirs { + if raw, err := os.ReadFile(filepath.Join(tree, d, "go.mod")); err == nil { + modules[d] = modulePath(string(raw)) + } + } + for _, p := range patterns { + if strings.HasPrefix(p, "./") || p == "." { + for _, d := range dirs { + out[d] = append(out[d], p) + } + continue + } + owner, best := "", -1 + for _, d := range dirs { + mp := modules[d] + if mp != "" && (p == mp || strings.HasPrefix(p, mp+"/")) && len(mp) > best { + owner, best = d, len(mp) + } + } + if best < 0 { + return nil, fmt.Errorf("no module of the workspace holds the package %s", p) + } + out[owner] = append(out[owner], p) + } + return out, nil +} + +// modulePath is the module path a go.mod declares, "" when it declares none. +func modulePath(gomod string) string { + for _, l := range strings.Split(gomod, "\n") { + if f := strings.Fields(l); len(f) == 2 && f[0] == "module" { + return strings.Trim(f[1], `"`) + } + } + return "" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside.go new file mode 100644 index 00000000..4b839d02 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside.go @@ -0,0 +1 @@ +package beside diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside_test.go b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside_test.go new file mode 100644 index 00000000..e98b6b71 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/beside_test.go @@ -0,0 +1,22 @@ +package beside + +import ( + "os" + "path/filepath" + "testing" +) + +// A test of a repository that reads another one, as the controller's, the node-engine's, the SDK's and the catalogue's do. +func TestReadsTheSiblingBesideIt(t *testing.T) { + root := os.Getenv("MESH_CHECK_BESIDE") + if root == "" { + t.Fatal("the sibling is not beside this check: MESH_CHECK_BESIDE is not set") + } + raw, err := os.ReadFile(filepath.Join(root, "sibling", "data.txt")) + if err != nil { + t.Fatal(err) + } + if string(raw) != "from the sibling\n" { + t.Fatalf("data.txt = %q", raw) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/go.mod b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/go.mod new file mode 100644 index 00000000..206a6e1f --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/testdata/repos/beside/go.mod @@ -0,0 +1,3 @@ +module fixture/beside + +go 1.26 diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 896d6b95..842e5952 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 github.com/nats-io/nats-server/v2 v2.11.17 ) diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 08b42fa5..58480c49 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index 687dc6b5..fb8ccc79 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -36,6 +36,8 @@ const sdkPackage = "@novox/mesh-sdk" // WorkspaceAge is how long a workspace is kept. const WorkspaceAge = 24 * time.Hour +var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) // FetchResult is what a passed fetch step says. @@ -48,6 +50,15 @@ type FetchResult struct { // Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK // with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459). Refused []Refusal `json:"refused"` + // The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk + // commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory. + // A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is + // left at the trunk commit with nothing installed. + Base string `json:"base,omitempty"` + Merge string `json:"merge,omitempty"` + Conflicts []string `json:"conflicts,omitempty"` + Changed []string `json:"changed,omitempty"` + Trees map[string]ModuleTree `json:"trees,omitempty"` } // Refusal is a module a fetch did not install the packages of. @@ -68,6 +79,7 @@ func (m *Module) fetchTool() StepTool { "repository": map[string]any{"type": "string", "description": "/ on the forge"}, "commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"}, "ref": map[string]any{"type": "string", "description": "refs/pull//head when only a pull request's head carries the commit"}, + "base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"}, }, "required": []string{"repository", "commit"}, }, @@ -80,6 +92,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { repository, _ := args["repository"].(string) commit, _ := args["commit"].(string) ref, _ := args["ref"].(string) + base, _ := args["base"].(string) + if base != "" && !commitOK.MatchString(base) { + return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base) + } if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") { return nil, fmt.Errorf("%q is not a repository, /", repository) } @@ -94,15 +110,23 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran { return m.Run(ctx, append(m.baseEnv(), env...), name, a...) } - remote := "ssh://" + ForgeAlias + "/" + repository + ".git" - if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil { + remote := m.remote(repository) + if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil { return nil, err } + res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}} + if base != "" { + if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil { + return nil, err + } + if res.Merge == "" { + return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed + } + } dirs, err := m.typeScriptModules(ctx, tree) if err != nil { return nil, err } - res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}} for _, d := range dirs { res.Modules = append(res.Modules, d) did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go b/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go index 07fa1e5b..da417017 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/harness_test.go @@ -48,7 +48,7 @@ func (f *fakeMachine) run(_ context.Context, env []string, name string, args ... } } else if name == "git" { for _, a := range args { - if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) { + if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files", "merge", "diff"}, a) { key = "git " + a break } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index 1cc1460c..1f9e60d0 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -31,9 +31,13 @@ type Module struct { Toolchain *Toolchain // Work is the work directory, outside the home: a workspace is //tree. Run runs the commands of a // step (a fake in a test). Clock is the time the steps are timed by. - Work string - Run forgekey.Runner - Clock func() time.Time + Work string + Run forgekey.Runner + // RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of + // the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote). + RemoteBase string + FetchOptions []buildstep.Option + Clock func() time.Time // Toolchains and Steps are the Put of the states of those names. Toolchains, Steps buildstep.Put // ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562). diff --git a/modules/mesh-build-node/cmd/mesh-build-node/preview.go b/modules/mesh-build-node/cmd/mesh-build-node/preview.go new file mode 100644 index 00000000..f5090e63 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/preview.go @@ -0,0 +1,93 @@ +package main + +// The merge preview of a fetch (novox/hq ADR 0484 clause 3; issue 1676), as build_go_fetch answers it. The merge itself +// is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it. +func (m *Module) remote(repository string) string { + base := m.RemoteBase + if base == "" { + base = "ssh://" + ForgeAlias + "/" + } + return base + repository + ".git" +} + +// merge makes the merge preview of the head onto base and writes the answer into res. +func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResult, tree, remote, base, commit string) error { + mr, err := buildstep.MergeInto(ctx, run, tree, remote, base, commit, m.FetchOptions...) + if err != nil { + return err + } + res.Base = mr.Base + res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed + if res.Merge == "" { + return nil + } + res.Trees, err = m.moduleTrees(ctx, tree, res.Merge) + return err +} + +// ModuleTree is what the merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256 +// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs. +type ModuleTree struct { + Tree string `json:"tree"` + Manifest string `json:"manifest"` + Locks map[string]string `json:"locks,omitempty"` +} + +var lockFiles = []string{"go.sum", "package-lock.json"} + +// moduleTrees are the trees, in the merge commit, of every directory of the tree that holds a module.json; the workspace +// is at the merge, so the manifest and the locks are read from it. +func (m *Module) moduleTrees(ctx context.Context, tree, merge string) (map[string]ModuleTree, error) { + r := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "ls-files", "--", "module.json", "*/module.json") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + trees := map[string]ModuleTree{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l == "" { + continue + } + dir := filepath.ToSlash(filepath.Dir(l)) + spec := merge + ":" + dir + if dir == "." { + spec = merge + "^{tree}" + } + t := m.Run(ctx, m.baseEnv(), "git", "-C", tree, "rev-parse", spec) + if t.Status != 0 || t.Err != "" { + return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr)) + } + mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)} + raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(l))) + if err != nil { + return nil, err + } + mt.Manifest = sha256Hex(raw) + for _, lock := range lockFiles { + if raw, err := os.ReadFile(filepath.Join(tree, filepath.FromSlash(dir), lock)); err == nil { + if mt.Locks == nil { + mt.Locks = map[string]string{} + } + mt.Locks[lock] = sha256Hex(raw) + } + } + trees[dir] = mt + } + return trees, nil +} + +func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/preview_integration_test.go b/modules/mesh-build-node/cmd/mesh-build-node/preview_integration_test.go new file mode 100644 index 00000000..487eebfe --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/preview_integration_test.go @@ -0,0 +1,95 @@ +//go:build integration + +package main + +// The fetch step's merge on real git repositories (issue 1676); preview_test.go proves the commands with a fake runner. + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +func git(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} + +func TestRealFetchMergesAConflictAndAPlainFetch(t *testing.T) { + root := t.TempDir() + bare, work := filepath.Join(root, "forge", "o", "r.git"), filepath.Join(root, "work") + os.MkdirAll(bare, 0o755) + os.MkdirAll(work, 0o755) + git(t, bare, "init", "-q", "--bare") + git(t, work, "init", "-q", "-b", "main") + files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`) + files["a.txt"] = "one\ntwo\nthree\n" + for n, c := range files { + write(t, filepath.Join(work, n), c, 0o644) + } + git(t, work, "add", ".") + git(t, work, "commit", "-q", "-m", "fork") + fork := git(t, work, "rev-parse", "HEAD") + change := func(from, ref string, f map[string]string) string { + git(t, work, "checkout", "-q", "-B", "tmp", from) + for n, c := range f { + write(t, filepath.Join(work, n), c, 0o644) + } + git(t, work, "add", ".") + git(t, work, "commit", "-q", "-m", "c") + git(t, work, "push", "-q", "-f", bare, "HEAD:"+ref) + return git(t, work, "rev-parse", "HEAD") + } + trunk := change(fork, "refs/heads/main", map[string]string{"a.txt": "one\ntwo\nTRUNK\n"}) + clean := change(fork, "refs/pull/1/head", map[string]string{"modules/m/extra.ts": "export {}\n"}) + clash := change(fork, "refs/pull/2/head", map[string]string{"a.txt": "one\ntwo\nHEAD\n"}) + + m, _, _ := newMachine(t) + m.Run = forgekey.ExecRunner + m.RemoteBase = "file://" + filepath.Join(root, "forge") + "/" + m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()} + + out, err := m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk}) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + tree := filepath.Join(m.Work, r.Workspace, "tree") + if r.Merge == "" || strings.Join(r.Changed, ",") != "modules/m/extra.ts" || len(r.Conflicts) != 0 || len(r.Modules) != 1 { + t.Fatalf("clean: %+v", r) + } + if want := git(t, tree, "rev-parse", r.Merge+":modules/m"); r.Trees["modules/m"].Tree != want { + t.Errorf("tree = %s, want %s", r.Trees["modules/m"].Tree, want) + } + if got := git(t, tree, "rev-parse", "HEAD"); got != r.Merge { + t.Errorf("the workspace is at %s, want the merge %s", got, r.Merge) + } + + out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk}) + if err != nil { + t.Fatalf("a conflict failed the step: %v", err) + } + r = out.(FetchResult) + if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.txt" { + t.Fatalf("conflict: %+v", r) + } + if got := git(t, filepath.Join(m.Work, r.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk { + t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk) + } + + out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"}) + if err != nil || out.(FetchResult).Merge != "" || out.(FetchResult).Trees != nil { + t.Errorf("no base: %+v %v", out, err) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/preview_test.go b/modules/mesh-build-node/cmd/mesh-build-node/preview_test.go new file mode 100644 index 00000000..8547d35e --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/preview_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +const baseSHA = "abcdef0123456789abcdef0123456789abcdef01" + +func commandLines(f *fakeMachine) string { + var out []string + for _, c := range f.calls { + out = append(out, joined(c)) + } + return strings.Join(out, "\n") +} + +// 1676/AC1: with a base the fetch merges the head onto it through the SDK's merge (the check merge's identity and flags, +// hooks off), and answers the base, the merge commit, the changed paths and the tree, manifest and locks of each module +// directory; the TypeScript modules are installed from the merged tree. +func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) { + m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil) + f.answer["git diff"] = forgekey.Ran{Stdout: "modules/m/package.json\n"} + args := map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA} + out, err := m.fetch(context.Background(), args) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + if r.Base != baseSHA || r.Merge != commitSHA || strings.Join(r.Changed, ",") != "modules/m/package.json" || len(r.Conflicts) != 0 { + t.Errorf("result = %+v", r) + } + if mt, ok := r.Trees["modules/m"]; !ok || mt.Tree != commitSHA || mt.Manifest == "" { + t.Errorf("trees = %+v", r.Trees) + } + if len(r.Modules) != 1 || r.Modules[0] != "modules/m" { + t.Errorf("modules = %v", r.Modules) + } + for _, want := range []string{"fetch --no-tags origin " + baseSHA, "core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} { + if !strings.Contains(commandLines(f), want) { + t.Errorf("git was not asked %q:\n%s", want, commandLines(f)) + } + } +} + +// 1676/AC2: a conflict is an answer: the step passes with the files named, no merge commit, and nothing installed. +func TestFetchWithAConflictAnswersTheFilesAndInstallsNothing(t *testing.T) { + files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`) + files["modules/m/package-lock.json"] = `{"lockfileVersion":3}` + m, f := fetchMachine(t, files, nil) + f.answer["git merge"] = forgekey.Ran{Status: 1, Stderr: "CONFLICT\n"} + f.answer["git diff"] = forgekey.Ran{Stdout: "a.ts\nsub/b.ts\n"} + out, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA}) + if err != nil { + t.Fatalf("a conflict failed the step: %v", err) + } + r := out.(FetchResult) + if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.ts,sub/b.ts" || len(r.Changed) != 0 || len(npmCalls(f)) != 0 { + t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f))) + } + if !strings.Contains(commandLines(f), "merge --abort") { + t.Errorf("the merge was not undone:\n%s", commandLines(f)) + } +} + +// 1676/AC3: no base is today's fetch: no merge command, and none of the preview in the answer. +func TestFetchWithoutABaseMakesNoMerge(t *testing.T) { + m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil) + out, err := m.fetch(context.Background(), fetchArgs) + if err != nil { + t.Fatal(err) + } + r := out.(FetchResult) + if strings.Contains(commandLines(f), "merge") || strings.Contains(commandLines(f), "unshallow") || r.Base != "" || r.Merge != "" || r.Trees != nil { + t.Errorf("a fetch without a base merged: %+v\n%s", r, commandLines(f)) + } +} + +func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) { + m, f := fetchMachine(t, nil, nil) + if _, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": "main"}); err == nil || len(f.calls) != 0 { + t.Errorf("err = %v after %d commands", err, len(f.calls)) + } +} diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index 322bf6f2..9c0040be 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -2,4 +2,4 @@ module meshbuildnode go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 5d8e396c..13d2b728 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,2 +1,4 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=