Merge pull request 'The Go build module takes a repository's own unpublished modules from its tree (issue 1921)' (#618) from fix/1921-zero-pins-own-packages into main

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -179,7 +179,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str
 				return
 			}
 			var more []TakenVersion
-			more, errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins, req.Packages)
+			more, errs[i] = m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages)
 			taken[i] = append(taken[i], more...)
 		}()
 	}
@@ -199,7 +199,7 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str
 // that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a
 // failure that names no such pin is the first failure. A requirement at the zero version is taken from the asked
 // versions (pins.go takeZeroPins).
-func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex, asked delivery.PackageVersions) ([]TakenVersion, error) {
+func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d string, pins *sync.Mutex, asked delivery.PackageVersions) ([]TakenVersion, error) {
 	var taken []TakenVersion
 	for {
 		failure := m.downloadAndVerify(ctx, dir, d)
@@ -207,7 +207,7 @@ func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins
 			return taken, nil
 		}
 		// A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest.
-		took, err := m.takeZeroPins(ctx, root, dir, d, failure.Error(), asked, pins)
+		took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins)
 		taken = append(taken, took...)
 		if err != nil {
 			return taken, err

modules/mesh-build-go/cmd/mesh-build-go/pins.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"maps"
 	"os"
 	"path/filepath"
@@ -257,7 +258,7 @@ func absoluteReplaces(body []byte, dir string) []byte {
 // the package's repository among the asked versions (the check's or try's members: another package of the same
 // repository names it); else the latest published version, the trunk's head. It never looks for the commit
 // 000000000000 among the pull request heads. It answers what it took; none when no zero requirement is named.
-func (m *Module) takeZeroPins(ctx context.Context, root, dir, d, failure string, asked delivery.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) {
+func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, failure string, asked delivery.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) {
 	raw, err := os.ReadFile(filepath.Join(dir, "go.mod"))
 	if err != nil || !strings.Contains(failure, zeroShort) {
 		return nil, nil
@@ -282,10 +283,27 @@ func (m *Module) takeZeroPins(ctx context.Context, root, dir, d, failure string,
 			continue // taken before
 		}
 		named := "v0.0.0-00010101000000-" + zeroShort
+		if "novox/"+p.repo == repository {
+			// A module of the checked repository itself: it is in the fetched tree and nowhere else, since nothing of the
+			// repository is published before its merge. Every module of the tree goes in, as one may require another.
+			own, err := m.useOwnModules(dir, root, p.repo)
+			if err != nil {
+				return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err)
+			}
+			if !slices.Contains(own, p.module) {
+				return taken, fmt.Errorf("the requirement of %s at the zero version: no module of that path is in the fetched tree of %s, and nothing of a repository is published before its merge", p.module, repository)
+			}
+			tv := TakenVersion{Dir: d, Package: p.module, Named: named, Built: "the fetched tree", Said: "a module of the checked repository: built from its tree"}
+			if m.Say != nil {
+				m.Say("%s: %s names no commit; built from the fetched tree of %s", d, p.module, repository)
+			}
+			taken = append(taken, tv)
+			continue
+		}
 		tv := TakenVersion{Dir: d, Package: p.module, Asked: askedFor(asked, p), Named: named}
 		why := "the ask names it"
 		if tv.Asked == "" {
-			tv.Asked, why = "latest", "the ask names no commit of its repository, so the trunk's latest published version"
+			tv.Asked, why = "latest", "the ask names no commit of its repository, so its latest published version"
 		} else if _, ok := asked[p.module]; !ok {
 			why = "the head of its repository among the asked versions"
 		}
@@ -314,3 +332,61 @@ func askedFor(asked delivery.PackageVersions, p pin) string {
 	}
 	return ""
 }
+
+// useOwnModules replaces, in dir's temporary modfile, every module of the fetched tree whose path lies in the repository
+// repo by its directory in the tree (once), and answers the paths of those modules. A replace of a module dir does not
+// require is harmless to Go; one module of a repository requires another of it at the zero version.
+func (m *Module) useOwnModules(dir, root, repo string) ([]string, error) {
+	tree := filepath.Join(root, "tree")
+	prefix := forgePrefix + repo
+	type found struct{ module, where string }
+	var mods []found
+	err := filepath.WalkDir(tree, func(path string, e fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if e.IsDir() {
+			if n := e.Name(); n == ".git" || n == "vendor" || n == "node_modules" {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if e.Name() != "go.mod" {
+			return nil
+		}
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		for _, line := range strings.Split(string(raw), "\n") {
+			if f := strings.Fields(line); len(f) == 2 && f[0] == "module" {
+				if mod := strings.Trim(f[1], `"`); (mod == prefix || strings.HasPrefix(mod, prefix+"/")) && filepath.Dir(path) != dir {
+					mods = append(mods, found{mod, filepath.Dir(path)})
+				}
+				break
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	modfile, _ := m.pinModfile(dir)
+	done := ""
+	if modfile != "" {
+		if b, err := os.ReadFile(modfile); err == nil {
+			done = string(b)
+		}
+	}
+	var paths []string
+	for _, f := range mods {
+		paths = append(paths, f.module)
+		if strings.Contains(done, "replace "+f.module+" =>") {
+			continue
+		}
+		if err := m.replaceInModfile(dir, f.module, f.where); err != nil {
+			return nil, err
+		}
+	}
+	return paths, nil
+}

modules/mesh-build-go/cmd/mesh-build-go/pins_test.go

@@ -310,6 +310,8 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) {
 			download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`},
 		{name: "the ask names another package of its repository", packages: map[string]any{"git.novox.be/novox/mesh-sdk/other": trunk}, want: sdk + "@" + trunk,
 			download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`},
+		{name: "the ask names the root of its repository, which no package of is published", packages: map[string]any{"git.novox.be/novox/mesh-sdk": trunk}, want: sdk + "@" + trunk,
+			download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`},
 		{name: "the ask names the head of a pull request", packages: map[string]any{sdk: commitSHA}, want: sdk + "@" + commitSHA, head: true},
 		{name: "the ask names nothing of its repository", packages: map[string]any{"git.novox.be/novox/mesh-tools/go": trunk}, want: sdk + "@latest",
 			download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`},
@@ -358,3 +360,42 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) {
 		})
 	}
 }
+
+// A requirement at the zero version of a module of the checked repository itself (a nested module not yet published, which
+// another module of the repository requires) is built from the fetched tree, and never asked of Go or the forge (issue 1921).
+func TestARequirementAtTheZeroVersionOfTheCheckedRepositorysOwnModuleIsTakenFromItsTree(t *testing.T) {
+	const own = "git.novox.be/novox/mesh-delivery/nested/go"
+	zeroMod := "module fixture/member\n\ngo 1.26\n\nrequire (\n\t" + own + " v0.0.0-00010101000000-" + zeroShort + "\n)\n"
+	h, states := newPinHarness(t, zeroMod, false, pullList)
+	base := h.m.Run
+	h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		line := name + " " + strings.Join(args, " ")
+		if name == "go" && strings.Contains(line, " mod download") && !strings.Contains(strings.Join(env, " "), "-modfile=") {
+			trees, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "tree"))
+			for _, tree := range trees {
+				os.MkdirAll(filepath.Join(tree, "nested", "go"), 0o755)
+				os.WriteFile(filepath.Join(tree, "nested", "go", "go.mod"), []byte("module "+own+"\n\ngo 1.26\n"), 0o644)
+			}
+			return forgekey.Ran{Status: 1, Stderr: "go: " + own + "@v0.0.0-00010101000000-" + zeroShort + ": invalid version: unknown revision " + zeroShort + "\n"}
+		}
+		return base(ctx, env, name, args...)
+	}
+	if _, err := fetchTool(t, h.m)(map[string]any{"repository": "novox/mesh-delivery", "commit": commitSHA}); err != nil {
+		t.Fatal(err)
+	}
+	st := states.waitEnd(t)
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	modfiles, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "pins", "*", "go.mod"))
+	if len(modfiles) != 1 {
+		t.Fatalf("modfiles = %v", modfiles)
+	}
+	body, _ := os.ReadFile(modfiles[0])
+	if !strings.Contains(string(body), "replace "+own+" => "+filepath.Join(filepath.Dir(filepath.Dir(filepath.Dir(modfiles[0]))), "tree", "nested", "go")) {
+		t.Errorf("the module of the tree was not replaced into the modfile:\n%s", body)
+	}
+	if h.ran(" mod download -json ") || h.ran("ls-remote") {
+		t.Error("a module of the checked repository was asked of Go or looked for among the pull request heads")
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/versions.go

@@ -107,7 +107,7 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer
 		failure := strings.TrimSpace(info.Error + "\n" + r.Err + "\n" + r.Stderr)
 		if !commitOK.MatchString(tv.Asked) {
 			// "latest" (the trunk's head) is no commit of a pull request: there is nothing to place.
-			return fmt.Errorf("the %s version of %s could not be taken (a download failure that may be retaken): %s", tv.Asked, tv.Package, causeLine(failure))
+			return fmt.Errorf("go cannot fetch the latest version of %s, and the ask names no commit of its repository to build it at: either no version of it is published, or the forge did not answer: %s", tv.Package, causeLine(failure))
 		}
 		repo, sub, _ := strings.Cut(strings.TrimPrefix(tv.Package, forgePrefix), "/")
 		p := pin{module: tv.Package, repo: repo, sub: sub, short: tv.Asked[:12]}