diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 04a18aadf..47d6586fd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -331,7 +331,13 @@ func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string, j if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil { return nil // a directory that vendors is read from its vendor directory, never through the module cache } - missing, err := ownFilesCover(dir, loaded) + // The files the steps build with: a directory with a temporary modfile (an asked version, a pin) is built with it and + // the sums beside it, which hold the requirements of the versions taken (takeRequirementsOf). + files := dir + if modfile, ok := m.pinModfile(dir); ok { + files = filepath.Dir(modfile) + } + missing, err := ownFilesCover(files, loaded) if err != nil { return fmt.Errorf("the go.mod and go.sum of %s: %w", d, err) } @@ -342,7 +348,7 @@ func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string, j return nil } -// ownFilesCover is what the go.mod and go.sum in dir lack for the modules its packages load, each as the line it lacks: +// ownFilesCover is what the go.mod and go.sum in dir (a directory's own, or those of its temporary modfile) lack for the modules its packages load, each as the line it lacks: // a mesh module (of the forge) required, the line it lacks naming the zero version (ADR 0573 §2; one named at a commit // is built against the asked version all the same, §4, and needs no lookup), every other required at the version the // load selected, with its two go.sum lines (the module's and its go.mod's) for the version Go reads it at. A module diff --git a/modules/mesh-build-go/cmd/mesh-build-go/judge_modfile_test.go b/modules/mesh-build-go/cmd/mesh-build-go/judge_modfile_test.go new file mode 100644 index 000000000..8d9ad6fc4 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/judge_modfile_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// A directory built with a temporary modfile (an asked version, a pin) is judged by the modfile and the sums beside it, the +// files its steps build with, and not by its own go.mod and go.sum: the requirements of an asked version of a mesh package +// are in the modfile (takeRequirementsOf), and the directory's own files lack them. +func TestADirectoryWithAModfileIsJudgedByTheModfileAndItsSums(t *testing.T) { + const nats = "github.com/nats-io/nats.go" + for _, c := range []struct { + name, modfile, sums string + lacks bool + }{ + {name: "the modfile names it with its sums", modfile: "require " + nats + " v1.54.0\n", sums: nats + " v1.54.0 h1:a=\n" + nats + " v1.54.0/go.mod h1:b=\n"}, + {name: "the modfile lacks it", modfile: "", sums: "", lacks: true}, + {name: "the modfile names it and the sums lack it", modfile: "require " + nats + " v1.54.0\n", sums: nats + " v1.54.0 h1:a=\n", lacks: true}, + } { + t.Run(c.name, func(t *testing.T) { + m := &Module{Work: t.TempDir()} + dir := filepath.Join(m.Work, "ws", "tree", "mod") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + head := "module fixture/mod\n\ngo 1.26\n\n" + // The directory's own files lack the requirement, whatever the modfile holds. + os.WriteFile(filepath.Join(dir, "go.mod"), []byte(head), 0o644) + pins := m.pinsDir(dir) + os.MkdirAll(pins, 0o755) + os.WriteFile(filepath.Join(pins, "go.mod"), []byte(head+c.modfile), 0o644) + os.WriteFile(filepath.Join(pins, "go.sum"), []byte(c.sums), 0o644) + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + return forgekey.Ran{Stdout: loadedModuleMark + nats + " v1.54.0\n"} + } + err := m.shutOffLoad(context.Background(), nil, dir, "mod", true) + if c.lacks != (err != nil) || (err != nil && !strings.Contains(err.Error(), "lacking")) { + t.Errorf("shutOffLoad = %v, want a refusal naming what is lacking: %v", err, c.lacks) + } + }) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions.go b/modules/mesh-build-go/cmd/mesh-build-go/versions.go index aa51c4f82..b54f52c96 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions.go @@ -53,6 +53,7 @@ type downloadInfo struct { Version string Sum string GoModSum string + GoMod string // the go.mod of the version in the module cache Error string } @@ -159,7 +160,10 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer return fmt.Errorf("the asked version %s of %s: %w", tv.Asked, tv.Package, err) } tv.Built, tv.Said = tv.Asked, "Go cannot fetch it: placed from the pull request's head, which has no checksum" - return nil + if _, err := os.Stat(filepath.Join(where, "go.mod")); err != nil { + return nil + } + return m.takeRequirementsOf(ctx, pinsDir, filepath.Join(where, "go.mod")) } if tagged && compareVersions(tv.Named, info.Version) > 0 { tv.Said = fmt.Sprintf("go.mod names %s, newer than the group's tagged version %s, which is built: the group's check is of its tagged version (ADR 0607 §3)", tv.Named, info.Version) @@ -177,12 +181,21 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer return fmt.Errorf("the asked version %s of %s could not be required: %s", info.Version, tv.Package, causeLine(r.Err+"\n"+r.Stderr)) } // The checksums Go computed of the download, which go mod verify checks the module cache against. + if err := appendSums(pinsDir, tv.Package, info); err != nil { + return err + } + // The requirements of that version go in too: the modfile names every module its packages load (ownFilesCover). + return m.takeRequirementsOf(ctx, pinsDir, info.GoMod) +} + +// appendSums writes the checksums of one download (the module's and its go.mod's) into the go.sum in pinsDir. +func appendSums(pinsDir, module string, info downloadInfo) error { sums := "" if info.Sum != "" { - sums += tv.Package + " " + info.Version + " " + info.Sum + "\n" + sums += module + " " + info.Version + " " + info.Sum + "\n" } if info.GoModSum != "" { - sums += tv.Package + " " + info.Version + "/go.mod " + info.GoModSum + "\n" + sums += module + " " + info.Version + "/go.mod " + info.GoModSum + "\n" } f, err := os.OpenFile(filepath.Join(pinsDir, "go.sum"), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o600) if err != nil { @@ -195,6 +208,46 @@ func (m *Module) takeVersion(ctx context.Context, root, dir string, tv *TakenVer return err } +// takeRequirementsOf requires, in the modfile of pinsDir, each module the go.mod at gomodPath (the go.mod of a version just +// taken, from what the fetch downloaded) requires at a version the modfile lacks or names older, with the checksums of +// each: the graph of a package version taken into a modfile is the version's, not the one go.mod was written against (a +// module whose go.mod names mesh-sdk v0.1.13 and is built at a newer one loads what the newer requires). A module of the +// forge is left to its own rule (the zero version, a pin, an asked version). +func (m *Module) takeRequirementsOf(ctx context.Context, pinsDir, gomodPath string) error { + if gomodPath == "" { + return nil + } + raw, err := os.ReadFile(gomodPath) + if err != nil { + return fmt.Errorf("the go.mod of the version taken: %w", err) + } + have, err := os.ReadFile(filepath.Join(pinsDir, "go.mod")) + if err != nil { + return err + } + want, _ := goModRequirements(string(raw)) + named, _ := goModRequirements(string(have)) + for _, module := range slices.Sorted(maps.Keys(want)) { + version := want[module] + if strings.HasPrefix(module, forgePrefix) || (named[module] != "" && compareVersions(named[module], version) >= 0) { + continue + } + r := m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "download", "-json", module+"@"+version) + var info downloadInfo + _ = json.Unmarshal([]byte(r.Stdout), &info) + if r.Status != 0 || r.Err != "" || info.Version == "" || info.Error != "" { + return downloadFailure(strings.TrimSpace(info.Error+"\n"+r.Err+"\n"+r.Stderr), "downloading %s@%s, a requirement of the version taken", module, version) + } + if r := m.Run(ctx, m.goEnv(), "go", "-C", pinsDir, "mod", "edit", "-require="+module+"@"+info.Version); r.Status != 0 || r.Err != "" { + return fmt.Errorf("%s@%s, a requirement of the version taken, could not be required: %s", module, info.Version, causeLine(r.Err+"\n"+r.Stderr)) + } + if err := appendSums(pinsDir, module, info); err != nil { + return err + } + } + return nil +} + // goModRequirements are the module paths a go.mod requires, with the version each names, and those it replaces. func goModRequirements(gomod string) (required map[string]string, replaced map[string]bool) { required, replaced = map[string]string{}, map[string]bool{} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go index 40f799197..0e591987d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/versions_test.go @@ -30,10 +30,17 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { forgeDown bool // the forge's listing of pull request heads fails too passing bool // the step ends failed as a transport fault, which the bus gives again (ADR 0664 §1) downloads int // the downloads asked in all (0: not counted) + requires string // the go.mod of the version taken, its requirements to be taken too }{ {name: "a trunk commit newer than go.mod's", asked: trunk, answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261101000000-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod="}`, built: "v0.1.15-0.20261101000000-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.15-0.20261101000000-aaaaaaaaaaaa"}, + // The requirements of the version taken go into the modfile with their sums, but a module of the forge is left to its + // own rule and one the modfile names at or above the version is left as it is. + {name: "a trunk commit whose go.mod requires more", asked: trunk, + answer: `{"Path":"` + sdk + `","Version":"v0.1.16-0.20261116170327-aaaaaaaaaaaa","Sum":"h1:zip=","GoModSum":"h1:mod=","GoMod":"GOMOD"}`, + built: "v0.1.16-0.20261116170327-aaaaaaaaaaaa", edit: "-require=" + sdk + "@v0.1.16-0.20261116170327-aaaaaaaaaaaa", + requires: "module " + sdk + "\n\ngo 1.26\n\nrequire (\n\tgithub.com/nats-io/nats.go v1.54.0\n\tgithub.com/klauspost/compress v1.20.0\n\tgit.novox.be/novox/mesh-contracts/go/x v0.1.0\n)\n"}, {name: "go.mod names a newer one", asked: trunk, answer: `{"Path":"` + sdk + `","Version":"v0.1.15-0.20261001000000-aaaaaaaaaaaa","Sum":"h1:zip="}`, built: "v0.1.15-0.20261022120000-" + pinShort, said: "newer than the asked"}, @@ -96,11 +103,20 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { if name == "git" && strings.Contains(line, "ls-remote") && c.forgeDown { return forgekey.Ran{Status: 128, Stderr: "fatal: unable to access the forge: Could not resolve host\n"} } + if name == "go" && strings.Contains(line, " mod download -json github.com/") { + module, version, _ := strings.Cut(strings.Fields(line)[len(strings.Fields(line))-1], "@") + return forgekey.Ran{Stdout: `{"Path":"` + module + `","Version":"` + version + `","Sum":"h1:` + module + `zip=","GoModSum":"h1:` + module + `mod="}`} + } if name == "go" && strings.Contains(line, " mod download -json "+sdk+"@") { h.mu.Lock() h.cmds = append(h.cmds, cmdRecord{env: env, line: line}) h.mu.Unlock() downloads++ + if c.requires != "" { + gomod := filepath.Join(t.TempDir(), "go.mod") + os.WriteFile(gomod, []byte(c.requires), 0o644) + return forgekey.Ran{Stdout: strings.ReplaceAll(c.answer, "GOMOD", gomod)} + } if c.failing { return forgekey.Ran{Status: 1, Stdout: c.answer} } @@ -158,6 +174,18 @@ func TestAnAskedVersionIsWrittenIntoTheModfileWhateverGoModNames(t *testing.T) { if placed := strings.Contains(string(body), "replace "+sdk+" => "); placed != c.failing { t.Errorf("the modfile replaces mesh-sdk by a placed head: %v, want %v:\n%s", placed, c.failing, body) } + if c.requires != "" { + for _, dep := range []string{"github.com/nats-io/nats.go@v1.54.0", "github.com/klauspost/compress@v1.20.0"} { + module, _, _ := strings.Cut(dep, "@") + if !h.ran("-require="+dep) || !strings.Contains(string(sums), module+" "+strings.TrimPrefix(dep, module+"@")+" h1:"+module+"zip=") || + !strings.Contains(string(sums), module+" "+strings.TrimPrefix(dep, module+"@")+"/go.mod h1:"+module+"mod=") { + t.Errorf("the requirement %s of the version taken was not required with its sums:\n%s", dep, sums) + } + } + if h.ran("mesh-contracts/go/x") { + t.Error("a module of the forge required by the version taken was looked for as a requirement") + } + } if _, other := h.m.pinModfile(filepath.Join(h.m.Work, res.Workspace, "tree", "other")); other { t.Error("a go.mod that requires no asked package was handed a modfile") }