diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 1a17af061..8eb13073a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -179,6 +179,11 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str return } var more []TakenVersion + // Then what only a module the directory replaces names, which the download would not miss (pins.go). + more, errs[i] = m.takeReplacedZeroPins(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins) + if taken[i] = append(taken[i], more...); errs[i] != nil { + return + } more, errs[i] = m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages) taken[i] = append(taken[i], more...) }() diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins.go b/modules/mesh-build-go/cmd/mesh-build-go/pins.go index 8563ba96a..19c1ec08a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins.go @@ -259,16 +259,117 @@ func absoluteReplaces(body []byte, dir string) []byte { return []byte(strings.Join(lines, "\n")) } -// takeZeroPins takes, for each requirement of dir's go.mod at the zero version that a go failure names, the version it is -// built against (ADR 0573 §2, §4: whatever go.mod names): the commit the ask names for the package; else the head of -// the package's repository among the asked versions (the check's or try's members: another package of the same -// repository names it); else the latest published version, the trunk's head. It never looks for the commit -// 000000000000 among the pull request heads. It answers what it took; none when no zero requirement is named. -func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, failure string, asked sdkstep.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { +// zeroRequirement is a requirement at the zero version in the module graph of a directory: by is "" when the directory's +// own go.mod names it, else the directory (relative to dir) of a module the directory replaces by a directory of the +// tree, whose go.mod names it. +type zeroRequirement struct { + pin + by string +} + +// zeroRequirementsOf are the requirements at the zero version of dir's module graph as Go builds it (ADR 0573 §2: +// whatever go.mod names): those of dir's go.mod, then those of each module dir's go.mod replaces by a directory +// (./x, ../x), each module once, own first. A module dir replaces is left out, since its replace decides it; a replaced +// module's own replaces are not followed, since Go reads replaces only of the directory it builds (issue 1966: buildstep, +// replaced by ../mesh-build-go/buildstep, imports a seat package mesh-build-image's go.mod did not name). +func zeroRequirementsOf(dir string) []zeroRequirement { raw, err := os.ReadFile(filepath.Join(dir, "go.mod")) - if err != nil || !strings.Contains(failure, zeroShort) { + if err != nil { + return nil + } + _, replaced := goModRequirements(string(raw)) + seen := map[string]bool{} + var out []zeroRequirement + add := func(gomod, by string) { + for _, p := range forgePins(gomod) { + if p.short != zeroShort || seen[p.module] || (by != "" && replaced[p.module]) { + continue + } + seen[p.module] = true + out = append(out, zeroRequirement{pin: p, by: by}) + } + } + add(string(raw), "") + local := localReplaces(string(raw)) + for _, module := range slices.Sorted(maps.Keys(local)) { + by := local[module] + b, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(by), "go.mod")) + if err != nil { + continue // a replace to a directory with no go.mod: Go says so in its own words + } + add(string(b), by) + } + return out +} + +// localReplaces are the replaces of a go.mod to a directory (., .., ./x, ../x), by module: the directory as written. +func localReplaces(gomod string) map[string]string { + out := map[string]string{} + inBlock := false + for _, line := range strings.Split(gomod, "\n") { + if i := strings.Index(line, "//"); i >= 0 { + line = line[:i] + } + f := strings.Fields(line) + switch { + case len(f) == 0: + continue + case inBlock && f[0] == ")": + inBlock = false + continue + case !inBlock && f[0] == "replace" && len(f) == 2 && f[1] == "(": + inBlock = true + continue + case !inBlock && f[0] != "replace": + continue + case !inBlock: + f = f[1:] + } + at := slices.Index(f, "=>") + if at < 1 || at+1 >= len(f) { + continue + } + target := strings.Trim(f[at+1], `"`) + if target == "." || target == ".." || strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") { + out[strings.Trim(f[0], `"`)] = filepath.ToSlash(filepath.Clean(target)) + } + } + return out +} + +// takeZeroPins takes, for each requirement at the zero version of dir's module graph (zeroRequirementsOf) when a go +// failure names the zero version, the version it is built against (takeZeroRequirements). It answers what it took; none +// when no zero requirement is named. +func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, failure string, asked sdkstep.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { + if !strings.Contains(failure, zeroShort) { return nil, nil } + return m.takeZeroRequirements(ctx, root, repository, dir, d, zeroRequirementsOf(dir), asked, pins) +} + +// takeReplacedZeroPins takes, before anything is downloaded, the requirements at the zero version that only a module +// dir replaces by a directory names (zeroRequirementsOf): go mod download loads only what dir's own go.mod names, so it +// passes without them, and the shut-off steps would look them up on the network (issue 1966). One the asked versions +// took (before, takeVersions) is left as it is. +func (m *Module) takeReplacedZeroPins(ctx context.Context, root, repository, dir, d string, asked sdkstep.PackageVersions, before []TakenVersion, pins *sync.Mutex) ([]TakenVersion, error) { + var replaced []zeroRequirement + for _, r := range zeroRequirementsOf(dir) { + if r.by != "" && !slices.ContainsFunc(before, func(tv TakenVersion) bool { return tv.Package == r.module }) { + replaced = append(replaced, r) + } + } + if len(replaced) == 0 { + return nil, nil + } + return m.takeZeroRequirements(ctx, root, repository, dir, d, replaced, asked, pins) +} + +// takeZeroRequirements takes, for each requirement at the zero version, the version it is built against (ADR 0573 §2, +// §4: whatever go.mod names): the commit the ask names for the package; else the head of the package's repository among +// the asked versions (the check's or try's members: another package of the same repository names it); else the latest +// published version, the trunk's head. It never looks for the commit 000000000000 among the pull request heads. One +// taken before (dir's modfile requires or replaces it) is left as it is. +func (m *Module) takeZeroRequirements(ctx context.Context, root, repository, dir, d string, reqs []zeroRequirement, asked sdkstep.PackageVersions, pins *sync.Mutex) ([]TakenVersion, error) { resolved := map[string]string{} if modfile, ok := m.pinModfile(dir); ok { if b, err := os.ReadFile(modfile); err == nil { @@ -281,14 +382,16 @@ func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, fai } } var taken []TakenVersion - for _, p := range forgePins(string(raw)) { - if p.short != zeroShort { - continue - } + for _, r := range reqs { + p := r.pin if v := resolved[p.module]; v != "" && !strings.HasSuffix(v, zeroShort) { continue // taken before } named := "v0.0.0-00010101000000-" + zeroShort + namedBy := "" + if r.by != "" { + namedBy = "; " + namedBySaid(r.by) + } if "novox/"+p.repo == repository { // A module of the checked repository itself: it is in the fetched tree and nowhere else, since nothing of the // repository is published before its merge. Every module of the tree goes in, as one may require another. @@ -299,7 +402,7 @@ func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, fai if !slices.Contains(own, p.module) { return taken, fmt.Errorf("the requirement of %s at the zero version: no module of that path is in the fetched tree of %s, and nothing of a repository is published before its merge", p.module, repository) } - tv := TakenVersion{Dir: d, Package: p.module, Named: named, Built: "the fetched tree", Said: "a module of the checked repository: built from its tree"} + tv := TakenVersion{Dir: d, Package: p.module, Named: named, Built: "the fetched tree", Said: "a module of the checked repository: built from its tree" + namedBy} if m.Say != nil { m.Say("%s: %s names no commit; built from the fetched tree of %s", d, p.module, repository) } @@ -316,7 +419,7 @@ func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, fai if err := m.takeVersion(ctx, root, dir, &tv, pins); err != nil { return taken, fmt.Errorf("the requirement of %s at the zero version: %w", p.module, err) } - tv.Said = strings.TrimPrefix(tv.Said+"; "+why, "; ") + tv.Said = strings.TrimPrefix(tv.Said+"; "+why+namedBy, "; ") if m.Say != nil { m.Say("%s: %s names no commit; built at %s (%s)", d, p.module, tv.Built, why) } @@ -325,6 +428,9 @@ func (m *Module) takeZeroPins(ctx context.Context, root, repository, dir, d, fai return taken, nil } +// namedBySaid says that a module dir replaces by the directory by names a package, and dir's go.mod does not. +func namedBySaid(by string) string { return "named by " + by + ", not by this go.mod" } + // askedFor is the commit the ask names for the package p: its own, else that of the first package of the same repository // in order (the members' heads are by repository), else none. func askedFor(asked sdkstep.PackageVersions, p pin) string { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index 62bec6fe7..9ce00ffea 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -305,6 +305,9 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { download string // what go mod download -json answers want string // the version asked of go head bool // the asked commit is on no trunk: only a pull request head carries it + // replaced: the directory's go.mod names no mesh package; a module it replaces by a directory of the tree requires + // it at the zero version, so go mod download passes and only a shut-off package load would look it up (1966/AC178-1) + replaced bool }{ {name: "the ask names the package", packages: map[string]any{sdk: trunk}, want: sdk + "@" + trunk, download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, @@ -316,13 +319,31 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { {name: "the ask names nothing of its repository", packages: map[string]any{"git.novox.be/novox/mesh-tools/go": trunk}, want: sdk + "@latest", download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, {name: "no ask", want: sdk + "@latest", download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, + {name: "named only by a module the directory replaces, the ask names it", packages: map[string]any{sdk: trunk}, want: sdk + "@" + trunk, replaced: true, + download: `{"Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip="}`}, + {name: "named only by a module the directory replaces, no ask", want: sdk + "@latest", replaced: true, + download: `{"Version":"v0.1.20-0.20261104000000-bbbbbbbbbbbb","Sum":"h1:zip="}`}, } { t.Run(c.name, func(t *testing.T) { - h, states := newPinHarness(t, zeroMod, false, pullList) + gomod := zeroMod + if c.replaced { + gomod = "module fixture/member\n\ngo 1.26\n\nrequire fixture/sib v0.0.0-00010101000000-" + zeroShort + "\n\nreplace fixture/sib => ./sib\n" + } + h, states := newPinHarness(t, gomod, false, pullList) base := h.m.Run h.m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { line := name + " " + strings.Join(args, " ") switch { + case c.replaced && name == "git" && strings.Contains(line, " checkout "): + r := base(ctx, env, name, args...) + trees, _ := filepath.Glob(filepath.Join(h.m.Work, "*", "tree")) + for _, tree := range trees { + os.MkdirAll(filepath.Join(tree, "sib"), 0o755) + os.WriteFile(filepath.Join(tree, "sib", "go.mod"), []byte(strings.Replace(zeroMod, "fixture/member", "fixture/sib", 1)), 0o644) + } + return r + case c.replaced && name == "go" && strings.Contains(line, " mod download") && !strings.Contains(line, "-json"): + return forgekey.Ran{} // go.mod names nothing missing: the download passes case name == "go" && strings.Contains(line, " mod download -json "): h.mu.Lock() h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) @@ -357,6 +378,13 @@ func TestARequirementAtTheZeroVersionIsTakenFromTheAskOrTheTrunk(t *testing.T) { if h.ran("pinned/mesh-sdk@" + zeroShort) { t.Error("a repository was placed at the commit of zeros") } + if c.replaced { + var res FetchResult + json.Unmarshal(st.Result, &res) + if len(res.Packages) != 1 || res.Packages[0].Package != sdk || !strings.Contains(res.Packages[0].Said, "named by sib, not by this go.mod") { + t.Errorf("the taken versions = %+v, want %s said as named by sib", res.Packages, sdk) + } + } }) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index a34b31747..8dbd79c6d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -66,6 +66,13 @@ func (m *Module) takeVersions(ctx context.Context, root, dir, d string, asked sd return nil, nil } required, replaced := goModRequirements(string(raw)) + // A package only a module dir replaces by a directory names is in dir's module graph too (zeroRequirementsOf). + namedBy := map[string]string{} + for _, r := range zeroRequirementsOf(dir) { + if _, ok := required[r.module]; !ok && r.by != "" { + required[r.module], namedBy[r.module] = zeroVersion, r.by + } + } var taken []TakenVersion for _, pkg := range slices.Sorted(maps.Keys(asked)) { named, ok := required[pkg] @@ -81,6 +88,9 @@ func (m *Module) takeVersions(ctx context.Context, root, dir, d string, asked sd return taken, err } } + if by := namedBy[pkg]; by != "" { + tv.Said = strings.TrimPrefix(tv.Said+"; "+namedBySaid(by), "; ") + } if m.Say != nil { m.Say("%s: %s is built at %s (asked %.12s, go.mod names %s)%s", d, pkg, tv.Built, tv.Asked, tv.Named, saidSuffix(tv.Said)) }