Kind: a version written into a workspace's modfile without what that version's graph needs, judged against files the build does not use. Builds: design 47 G (ADR 0573 §2, §4, as written in versions.go: asked versions hold whatever go.mod names)
742 lines
31 KiB
Go
742 lines
31 KiB
Go
package main
|
|
|
|
// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of
|
|
// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every
|
|
// go.mod in it names, downloaded into the module's cache and verified against go.sum.
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/mod/modfile"
|
|
|
|
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
|
)
|
|
|
|
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
|
|
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
|
|
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
|
const ForgeAlias = "mesh-forge"
|
|
|
|
var (
|
|
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
|
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
|
)
|
|
|
|
// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose
|
|
// Go modules it downloaded.
|
|
type FetchResult struct {
|
|
Workspace string `json:"workspace"`
|
|
Commit string `json:"commit"`
|
|
Modules []string `json:"modules"`
|
|
// The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the
|
|
// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
|
|
// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
|
|
// files, and the workspace is left at the trunk commit.
|
|
Base string `json:"base,omitempty"`
|
|
Merge string `json:"merge,omitempty"`
|
|
Conflicts []string `json:"conflicts,omitempty"`
|
|
Changed []string `json:"changed,omitempty"`
|
|
// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
|
|
Programs *[]steprun.Program `json:"programs,omitempty"`
|
|
Trees map[string]sdkstep.ModuleTree `json:"trees,omitempty"`
|
|
// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
|
|
// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
|
|
Beside []Sibling `json:"beside,omitempty"`
|
|
// Packages are, of each directory whose go.mod requires an asked mesh package, the version it is built against
|
|
// (versions.go; novox/hq ADR 0573 §2): the asked, or go.mod's when it names a newer one, said.
|
|
Packages []TakenVersion `json:"packages,omitempty"`
|
|
}
|
|
|
|
// fetchTool is the fetch step tool.
|
|
func (m *Module) fetchTool() StepTool {
|
|
return StepTool{
|
|
Name: "build_go_fetch",
|
|
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
|
"workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " +
|
|
"once; the step ends on its key in the steps state with the workspace's id.",
|
|
Input: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
|
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
|
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
|
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
|
"dir": sdkstep.FetchRequestProperties()["dir"],
|
|
"paths": sdkstep.FetchRequestProperties()["paths"],
|
|
"packages": sdkstep.FetchRequestProperties()["packages"],
|
|
"builds": map[string]any{"type": "array", "description": "optional: a group run's package members, each by its build's digest, read from the artifact store and laid as the package it names (design 47 G §465)",
|
|
"items": map[string]any{"type": "object", "properties": map[string]any{
|
|
"package": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"},
|
|
"artifact": map[string]any{"type": "string"}, "digest": map[string]any{"type": "string"},
|
|
}, "required": []string{"package", "module", "artifact", "digest"}}},
|
|
"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
|
|
"items": map[string]any{"type": "object", "properties": map[string]any{
|
|
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
|
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
|
}, "required": []string{"repository", "commit"}}},
|
|
},
|
|
"required": []string{"repository", "commit"},
|
|
},
|
|
Long: true,
|
|
Run: m.fetch,
|
|
}
|
|
}
|
|
|
|
// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466
|
|
// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it
|
|
// (issue 1839).
|
|
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
|
|
// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
|
|
// named paths, the workspace is narrowed to them (steprun.Narrow).
|
|
req, err := sdkstep.ParseFetchRequest(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// A fetch with no dir (the checker's check of a pull request, until G removes the checker; A-447e) fetches the tree
|
|
// whole: every go.mod of it.
|
|
siblings, err := siblingsArg(args["beside"])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
builds, err := steprun.BuildsArg(args["builds"])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
m.CleanWorkspaces(m.now())
|
|
id := sdkstep.NewID()
|
|
done := m.spaces().Hold(id)
|
|
defer func() {
|
|
done()
|
|
if err != nil {
|
|
if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
|
|
m.Say("the failed fetch's workspace %s: %v", id, rerr)
|
|
}
|
|
}
|
|
}()
|
|
return m.fetchInto(ctx, req, id, siblings, builds)
|
|
}
|
|
|
|
func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id string, siblings []siblingArg, builds []sdkstep.FetchBuild) (any, error) {
|
|
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
|
var err error
|
|
root := filepath.Join(m.Work, id)
|
|
tree := filepath.Join(root, "tree")
|
|
if err := os.MkdirAll(tree, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
remote := m.remote(repository)
|
|
if err := steprun.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil {
|
|
return nil, err
|
|
}
|
|
res := FetchResult{Workspace: id, Commit: commit}
|
|
if base != "" {
|
|
if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := m.dissociate(ctx, tree); err != nil {
|
|
return nil, err
|
|
}
|
|
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
|
|
return nil, err
|
|
}
|
|
if (base == "" || res.Merge != "") && len(req.Paths) > 0 {
|
|
if err := steprun.Narrow(tree, req); err != nil {
|
|
return nil, err
|
|
}
|
|
// Said beside the tree, so the steps that list the tree's go.mod files leave out those narrowed away.
|
|
if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
dirs, err := m.goModDirs(ctx, tree)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if dir := cleanDir(req.Dir); dir != "" {
|
|
// A fetch asked with a module's directory is of that Go module and what its go.mod replaces with a local path,
|
|
// never another Go module of the tree (design 47 G, A-447). Said beside the tree, so every later step lists the
|
|
// same modules.
|
|
if dirs, err = scopedDirs(tree, dir, dirs); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(root, scopeMark), []byte(strings.Join(dirs, "\n")), 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(root, moduleDirMark), []byte(dir), 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
res.Modules = dirs
|
|
if base != "" && res.Merge == "" {
|
|
return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
|
|
}
|
|
// A group run's package members are laid from their builds, never fetched by a version (builds.go, §465).
|
|
if err := m.layBuilds(ctx, root, dirs, builds); err != nil {
|
|
return nil, err
|
|
}
|
|
// The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the
|
|
// step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time.
|
|
var pins sync.Mutex
|
|
errs := make([]error, len(dirs))
|
|
taken := make([][]TakenVersion, len(dirs))
|
|
slots := make(chan struct{}, m.parallel())
|
|
var wg sync.WaitGroup
|
|
for i, d := range dirs {
|
|
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
|
|
continue // a module that vendors resolves nothing
|
|
}
|
|
wg.Add(1)
|
|
slots <- struct{}{}
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-slots }()
|
|
// The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go).
|
|
if taken[i], errs[i] = m.takeVersions(ctx, root, filepath.Join(tree, d), d, req.Packages, &pins); errs[i] != nil {
|
|
return
|
|
}
|
|
var more []TakenVersion
|
|
// Then what only a module the directory replaces names, which the download would not miss (pins.go).
|
|
more, errs[i] = m.takeReplacedZeroPins(ctx, root, repository, filepath.Join(tree, d), d, req.Packages, taken[i], &pins)
|
|
if taken[i] = append(taken[i], more...); errs[i] != nil {
|
|
return
|
|
}
|
|
more, errs[i] = m.downloadModules(ctx, root, repository, filepath.Join(tree, d), d, &pins, req.Packages)
|
|
taken[i] = append(taken[i], more...)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
for _, err := range errs {
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if err := m.downloadBeside(ctx, root, tree, dirs); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, t := range taken {
|
|
res.Packages = append(res.Packages, t...)
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module
|
|
// that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a
|
|
// failure that names no such pin is the first failure. A requirement at the zero version is taken from the asked
|
|
// versions (pins.go takeZeroPins).
|
|
func (m *Module) downloadModules(ctx context.Context, root, repository, dir, d string, pins *sync.Mutex, asked sdkstep.PackageVersions) ([]TakenVersion, error) {
|
|
var taken []TakenVersion
|
|
for {
|
|
failure := m.downloadAndVerify(ctx, dir, d)
|
|
if failure == nil {
|
|
// The proof that what was downloaded is all the shut-off steps load: else the fetch fails here, naming the
|
|
// package, and not a vet, a test or a build later with less to say (issue 1966).
|
|
return taken, m.shutOffLoad(ctx, m.goEnvFor(dir), dir, d, true)
|
|
}
|
|
// A requirement at the zero version names no commit: it is taken from the asked versions or the trunk's latest.
|
|
took, err := m.takeZeroPins(ctx, root, repository, dir, d, failure.Error(), asked, pins)
|
|
taken = append(taken, took...)
|
|
if err != nil {
|
|
return taken, err
|
|
}
|
|
pins.Lock()
|
|
progress, err := m.resolvePins(ctx, root, dir, failure.Error())
|
|
pins.Unlock()
|
|
if err != nil {
|
|
return taken, err
|
|
}
|
|
if !progress && len(took) == 0 {
|
|
return taken, failure
|
|
}
|
|
}
|
|
}
|
|
|
|
// shutOffLoadFormat is what go list -e says of each package it loads: nothing, or the package and Go's words of its error.
|
|
const shutOffLoadFormat = `{{with .Error}}{{$.ImportPath}}: {{.Err}}{{"\n"}}{{end}}`
|
|
|
|
// loadedModuleMark opens the line judgedLoadFormat says of the module a loaded package lies in.
|
|
const loadedModuleMark = "loaded-module "
|
|
|
|
// judgedLoadFormat is shutOffLoadFormat with, for each package of a module other than the directory's own, a line
|
|
// naming that module, its version and its replacement: what the go.mod and go.sum of the tree are judged against.
|
|
const judgedLoadFormat = shutOffLoadFormat + `{{with .Module}}{{if not .Main}}` + loadedModuleMark +
|
|
`{{.Path}} {{.Version}}{{with .Replace}} {{.Path}} {{.Version}}{{end}}{{"\n"}}{{end}}{{end}}`
|
|
|
|
// loadedModule is a module a package of the load lies in: its path and version, and the replacement Go read it from
|
|
// (a directory has no version).
|
|
type loadedModule struct{ path, version, replacePath, replaceVersion string }
|
|
|
|
// shutOffLoad loads the packages of dir, with their tests and every package they import, as a shut-off step does
|
|
// (GOPROXY=off, the step's own environment env), and fails naming each package whose load would reach the network, in
|
|
// Go's words. A package that fails to load for another cause (a compile error, an import of a package of the module
|
|
// that does not exist) is the shut-off steps' to say, with their own verdict lines, and is not the fetch's.
|
|
//
|
|
// A judged directory (one of the checked tree, not a sibling laid beside) is loaded twice, untagged and with the
|
|
// integration level's tags, and its own go.mod and go.sum must name every module either load reaches (ownFilesCover):
|
|
// a module the shut-off steps meet that the tree's files do not name is looked up in the checksum database, which no
|
|
// shut-off step reaches (try 2 of design 47's milestone G: restic's tests load nats-server through testbus, which
|
|
// neither its go.mod nor its go.sum named, and its go test failed verifying it). The fetch fails naming the module
|
|
// and the line its go.mod or go.sum lacks, for every repository the check builds.
|
|
func (m *Module) shutOffLoad(ctx context.Context, env []string, dir, d string, judged bool) error {
|
|
tagSets := [][]string{nil}
|
|
format := shutOffLoadFormat
|
|
if judged {
|
|
tagSets, format = append(tagSets, integrationTags), judgedLoadFormat
|
|
}
|
|
var lost []string
|
|
var loaded []loadedModule
|
|
for _, tags := range tagSets {
|
|
args := []string{"-C", dir, "list", "-e", "-deps", "-test"}
|
|
if len(tags) > 0 {
|
|
args = append(args, "-tags", strings.Join(tags, ","))
|
|
}
|
|
r := m.Run(ctx, append(env, "GOPROXY=off"), "go", append(args, "-f", format, "./...")...)
|
|
for _, l := range strings.Split(r.Stdout+"\n"+r.Stderr, "\n") {
|
|
l = strings.TrimSpace(l)
|
|
if rest, ok := strings.CutPrefix(l, loadedModuleMark); ok {
|
|
f := append(strings.Fields(rest), "", "", "")
|
|
loaded = append(loaded, loadedModule{path: f[0], version: f[1], replacePath: f[2], replaceVersion: f[3]})
|
|
} else if strings.Contains(l, "GOPROXY=off") && !slices.Contains(lost, l) {
|
|
lost = append(lost, l)
|
|
}
|
|
}
|
|
if len(lost) > 0 {
|
|
return fmt.Errorf("the packages of %s load shut off only with what the fetch did not download: %s", d, strings.Join(lost, "; "))
|
|
}
|
|
// A load that could not run at all proves nothing: the fetch fails here, not a step later with less to say.
|
|
if r.Err != "" || r.Status != 0 {
|
|
say := strings.TrimSpace(r.Err + " " + r.Stderr)
|
|
return fmt.Errorf("the shut-off package load of %s could not run (exit %d): %s", d, r.Status, say)
|
|
}
|
|
}
|
|
if !judged || len(loaded) == 0 {
|
|
return nil // a load that reaches no module beyond the directory's own has nothing to name
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
|
return nil // a directory that vendors is read from its vendor directory, never through the module cache
|
|
}
|
|
// The files the steps build with: a directory with a temporary modfile (an asked version, a pin) is built with it and
|
|
// the sums beside it, which hold the requirements of the versions taken (takeRequirementsOf).
|
|
files := dir
|
|
if modfile, ok := m.pinModfile(dir); ok {
|
|
files = filepath.Dir(modfile)
|
|
}
|
|
missing, err := ownFilesCover(files, loaded)
|
|
if err != nil {
|
|
return fmt.Errorf("the go.mod and go.sum of %s: %w", d, err)
|
|
}
|
|
if len(missing) > 0 {
|
|
return fmt.Errorf("the go.mod and go.sum of %s do not name every module its packages load (untagged and %s), so a shut-off step would look it up: lacking %s",
|
|
d, strings.Join(integrationTags, ","), strings.Join(missing, "; "))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ownFilesCover is what the go.mod and go.sum in dir (a directory's own, or those of its temporary modfile) lack for the modules its packages load, each as the line it lacks:
|
|
// a mesh module (of the forge) required, the line it lacks naming the zero version (ADR 0573 §2; one named at a commit
|
|
// is built against the asked version all the same, §4, and needs no lookup), every other required at the version the
|
|
// load selected, with its two go.sum lines (the module's and its go.mod's) for the version Go reads it at. A module
|
|
// replaced by a directory has no sums to lack.
|
|
func ownFilesCover(dir string, loaded []loadedModule) ([]string, error) {
|
|
gomod, err := os.ReadFile(filepath.Join(dir, "go.mod"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
required, _ := goModRequirements(string(gomod))
|
|
sums := map[string]bool{}
|
|
if raw, err := os.ReadFile(filepath.Join(dir, "go.sum")); err == nil {
|
|
for _, l := range strings.Split(string(raw), "\n") {
|
|
if f := strings.Fields(l); len(f) == 3 {
|
|
sums[f[0]+" "+f[1]] = true
|
|
}
|
|
}
|
|
} else if !errors.Is(err, os.ErrNotExist) {
|
|
return nil, err
|
|
}
|
|
var missing []string
|
|
lack := func(line string) {
|
|
if !slices.Contains(missing, line) {
|
|
missing = append(missing, line)
|
|
}
|
|
}
|
|
for _, l := range loaded {
|
|
if strings.HasPrefix(l.path, forgePrefix) {
|
|
if _, ok := required[l.path]; !ok {
|
|
lack("go.mod: require " + l.path + " " + zeroVersion)
|
|
}
|
|
continue
|
|
}
|
|
if required[l.path] != l.version {
|
|
lack("go.mod: require " + l.path + " " + l.version)
|
|
}
|
|
path, version := l.path, l.version
|
|
if l.replacePath != "" {
|
|
if l.replaceVersion == "" {
|
|
continue // a directory
|
|
}
|
|
path, version = l.replacePath, l.replaceVersion
|
|
}
|
|
for _, v := range []string{version, version + "/go.mod"} {
|
|
if !sums[path+" "+v] {
|
|
lack("go.sum: " + path + " " + v + " h1:")
|
|
}
|
|
}
|
|
}
|
|
return missing, nil
|
|
}
|
|
|
|
// verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once.
|
|
var verifyLocks sync.Map
|
|
|
|
// verifiedMark is the file naming a pass of go mod verify for a go.mod and go.sum pair, "" when a pass is not kept:
|
|
// the module cache is only read by steps that are shut off with it read-only (ADR 0448 §3), so the pass holds only then.
|
|
// A directory with a temporary modfile (an asked version, a pin on a head) verifies the modfile and the sums beside it,
|
|
// which a placed repository's path makes the workspace's own.
|
|
func (m *Module) verifiedMark(dir string) string {
|
|
if m.Cache == "" || m.Bubblewrap == nil {
|
|
return ""
|
|
}
|
|
from := dir
|
|
if modfile, ok := m.pinModfile(dir); ok {
|
|
from = filepath.Dir(modfile)
|
|
}
|
|
h := sha256.New()
|
|
for _, name := range []string{"go.mod", "go.sum"} {
|
|
raw, _ := os.ReadFile(filepath.Join(from, name))
|
|
fmt.Fprintf(h, "%s %d\n", name, len(raw))
|
|
h.Write(raw)
|
|
}
|
|
return filepath.Join(m.Cache, "verified", hex.EncodeToString(h.Sum(nil)))
|
|
}
|
|
|
|
func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
|
|
for _, verb := range []string{"download", "verify"} {
|
|
mark := ""
|
|
if verb == "verify" {
|
|
if mark = m.verifiedMark(dir); mark != "" {
|
|
mu, _ := verifyLocks.LoadOrStore(mark, &sync.Mutex{})
|
|
mu.(*sync.Mutex).Lock()
|
|
defer mu.(*sync.Mutex).Unlock()
|
|
if _, err := os.Stat(mark); err == nil {
|
|
continue // this go.mod and go.sum passed before
|
|
}
|
|
}
|
|
}
|
|
cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir)
|
|
if modfile, ok := m.pinModfile(dir); ok && verb == "verify" {
|
|
// go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories.
|
|
cmd, env = []string{"-C", filepath.Dir(modfile), "mod", verb}, m.goEnv()
|
|
}
|
|
r := m.Run(ctx, env, "go", cmd...)
|
|
if r.Status != 0 || r.Err != "" {
|
|
return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
|
|
}
|
|
if mark != "" {
|
|
// Said after the pass; a mark that cannot be written only means the next fetch verifies again.
|
|
if os.MkdirAll(filepath.Dir(mark), 0o700) == nil {
|
|
os.WriteFile(mark, nil, 0o600)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
|
|
// When the fetch was asked with a module's directory (scopeMark), the steps after it see only the directories it chose.
|
|
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
|
|
if b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), scopeMark)); err == nil {
|
|
return strings.Fields(string(b)), nil
|
|
}
|
|
r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
|
|
if r.Status != 0 || r.Err != "" {
|
|
return nil, fmt.Errorf("git ls-files: %s", dforgekey.Said(r.Err+"\n"+r.Stderr))
|
|
}
|
|
dirs := []string{}
|
|
for _, l := range strings.Split(r.Stdout, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" && !inTestdata(l) {
|
|
if narrowed(tree) {
|
|
if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(l))); err != nil {
|
|
continue // outside a narrowed workspace's paths (steprun.Narrow): no module of this build
|
|
}
|
|
}
|
|
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
|
|
}
|
|
}
|
|
sort.Strings(dirs)
|
|
return dirs, nil
|
|
}
|
|
|
|
// testDirs are the directories the unit and integration steps and the digest take: the chosen go.mod directories at or
|
|
// under the module's own directory. A directory the module's go.mod replaces from outside its own stays for download and
|
|
// resolution alone; its tests are another module's (design 47 G, A-447c). A workspace fetched with no dir keeps every go.mod.
|
|
func (m *Module) testDirs(ctx context.Context, tree string) ([]string, error) {
|
|
all, err := m.goModDirs(ctx, tree)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
own, ok := moduleDirOf(tree)
|
|
if !ok || own == "." || own == "" {
|
|
return all, nil
|
|
}
|
|
out := []string{}
|
|
for _, d := range all {
|
|
if d == own || strings.HasPrefix(d, own+"/") {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules
|
|
// of the tree (issue 1810).
|
|
func inTestdata(path string) bool {
|
|
for _, seg := range strings.Split(filepath.ToSlash(path), "/") {
|
|
if seg == "testdata" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// goEnv is what a go command of a step is given beyond the base environment: the installed go alone (GOTOOLCHAIN=local, ADR 0477 §1: no toolchain is ever downloaded),
|
|
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
|
|
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
|
|
// that vendors, see goEnvFor.
|
|
func (m *Module) goEnv() []string {
|
|
modcache := filepath.Join(m.Cache, "gomod")
|
|
if m.GoModCache != "" {
|
|
modcache = m.GoModCache
|
|
}
|
|
env := []string{
|
|
"GOTOOLCHAIN=local",
|
|
"GOFLAGS=-mod=mod",
|
|
"GOMODCACHE=" + modcache,
|
|
"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
|
|
}
|
|
if proxy := m.goSeat(); proxy != "" {
|
|
// The Go seat first, the public proxy second, the forge's git last (CAT-BG3, E11: direct goes in the first
|
|
// change after mesh-sdk's and mesh-contracts' first numbered versions are in the Go seat); no GOPRIVATE, and
|
|
// the checksum database off for the forge's paths, which go.sum and the builder's h1: digest guard instead.
|
|
// The seat goes on to the public proxy only on a 404 or 410 (",": a package it does not hold), so a seat that
|
|
// fails otherwise (5xx, refused, timeout) fails the fetch loudly; the public proxy goes on to direct on any
|
|
// failure ("|"): a public proxy the node cannot reach stopped the fetch before direct, and direct reaches the
|
|
// forge's git for a package the seat does not hold (issue 1966, ADR 0605 §6).
|
|
env = append(env, "GOPROXY="+proxy+","+PublicGoProxy+"|direct", "GONOSUMDB="+ForgeGoPaths, "GOPRIVATE=")
|
|
} else {
|
|
env = append(env, "GOPRIVATE="+ForgeGoPaths)
|
|
}
|
|
return append(append(env, "GIT_TERMINAL_PROMPT=0", "GIT_SSH_COMMAND=ssh -o BatchMode=yes"), m.threadEnv()...)
|
|
}
|
|
|
|
// PublicGoProxy is Go's public module proxy; ForgeGoPaths the module paths the forge serves.
|
|
const (
|
|
PublicGoProxy = "https://proxy.golang.org"
|
|
ForgeGoPaths = "git.novox.be"
|
|
)
|
|
|
|
// goSeat is the Go seat's proxy URL from its binding (`at`, and `serves` scheme, port and go-path), empty when the
|
|
// module holds no binding of it or the binding names no address.
|
|
func (m *Module) goSeat() string {
|
|
if m.GoSeatFile == "" {
|
|
return ""
|
|
}
|
|
raw, err := os.ReadFile(m.GoSeatFile)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
var b struct {
|
|
At string `json:"at"`
|
|
Serves struct {
|
|
Scheme string `json:"scheme"`
|
|
Port int `json:"port"`
|
|
GoPath string `json:"go-path"`
|
|
} `json:"serves"`
|
|
}
|
|
if json.Unmarshal(raw, &b) != nil || b.At == "" || b.Serves.GoPath == "" {
|
|
return ""
|
|
}
|
|
scheme := b.Serves.Scheme
|
|
if scheme == "" {
|
|
scheme = "http"
|
|
}
|
|
if b.Serves.Port == 0 {
|
|
return fmt.Sprintf("%s://%s%s", scheme, b.At, b.Serves.GoPath)
|
|
}
|
|
return fmt.Sprintf("%s://%s:%d%s", scheme, b.At, b.Serves.Port, b.Serves.GoPath)
|
|
}
|
|
|
|
// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
|
|
// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum.
|
|
func (m *Module) goEnvFor(dir string) []string {
|
|
env := m.goEnv()
|
|
if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
|
|
env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
|
|
} else if modfile, ok := m.pinModfile(dir); ok {
|
|
env = append(env, "GOFLAGS=-mod=mod -modfile="+modfile) // a pin on a pull request's head (pins.go)
|
|
}
|
|
return env
|
|
}
|
|
|
|
// CleanWorkspaces removes the workspaces no step holds whose last use is older than steprun.UnheldBound: the safety
|
|
// net for a caller that never released one (issue 1839, ADR 0509).
|
|
func (m *Module) CleanWorkspaces(now time.Time) {
|
|
m.sweepSaid(m.spaces().Sweep(now, steprun.UnheldBound))
|
|
m.SweepKept(now, KeptBound)
|
|
m.sweepKeptBuilds(now, KeptBound)
|
|
}
|
|
|
|
// clock is the time now: Now when a test set it, else the wall clock.
|
|
func (m *Module) clock() time.Time {
|
|
if m.Now != nil {
|
|
return m.Now()
|
|
}
|
|
return time.Now()
|
|
}
|
|
func (m *Module) now() time.Time { return m.clock() }
|
|
|
|
// causeLine is the first line of a go command's output that says something other than Go's own progress: `go:
|
|
// downloading ...` comes first when Go fetches a module, and `# <package>` heads a compile error of go build or go vet
|
|
// (issue 1703: the reason said only the package), and the cause of a failure on a later line.
|
|
//
|
|
// A cause Go writes over several lines keeps its indented continuation lines, joined by a space: a module graph error
|
|
// is "X requires" with the module that fails, and why, on the next line (issue 1966: the composer's reason ended at
|
|
// "requires" and hid "golang.org/x/tools@v0.42.0: module lookup disabled").
|
|
func causeLine(s string) string {
|
|
lines := strings.Split(s, "\n")
|
|
for i, l := range lines {
|
|
if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") && !strings.HasPrefix(l, "# ") {
|
|
return withContinuation(l, lines[i+1:])
|
|
}
|
|
}
|
|
return firstLine(s)
|
|
}
|
|
|
|
// withContinuation is first with each following line Go indented under it, trimmed and joined by a space.
|
|
func withContinuation(first string, rest []string) string {
|
|
for _, l := range rest {
|
|
if !strings.HasPrefix(l, "\t") && !strings.HasPrefix(l, " ") {
|
|
break
|
|
}
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
first += " " + l
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
func firstLine(s string) string {
|
|
for _, l := range strings.Split(s, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
return l
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
|
|
const narrowedMark = "narrowed"
|
|
|
|
// narrowed says whether the fetch narrowed the workspace whose tree this is.
|
|
func narrowed(tree string) bool {
|
|
_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
|
|
return err == nil
|
|
}
|
|
|
|
// scopeMark is the file beside a workspace's tree naming the Go module directories a fetch asked with a module's
|
|
// directory chose, one to a line (empty: none).
|
|
const scopeMark = "gomodules"
|
|
|
|
// moduleDirMark is the file beside a workspace's tree naming the module directory the fetch was asked with: the one
|
|
// module.json a step compares generated files for (design 47 G, A-447c).
|
|
const moduleDirMark = "moduledir"
|
|
|
|
// moduleDirOf is the module directory the workspace of tree was fetched for, ok false for a workspace with none.
|
|
func moduleDirOf(tree string) (string, bool) {
|
|
b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), moduleDirMark))
|
|
if err != nil {
|
|
return "", false
|
|
}
|
|
return string(b), true
|
|
}
|
|
|
|
// cleanDir is a request's module directory in the form goModDirs lists, "" for none.
|
|
func cleanDir(dir string) string {
|
|
if strings.TrimSpace(dir) == "" {
|
|
return ""
|
|
}
|
|
return strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/")
|
|
}
|
|
|
|
// scopedDirs are the go.mod directories of all that a module directory dir chooses: dir itself and each directory its
|
|
// go.mod replaces with a local path, followed through their own replaces (design 47 G, A-447), in order. A directory
|
|
// the tree holds no go.mod for is left out. A directory holding no go.mod chooses every go.mod under it, and one that
|
|
// chooses no Go module at all is an error naming it (A-447a).
|
|
func scopedDirs(tree, dir string, all []string) ([]string, error) {
|
|
have := map[string]bool{}
|
|
for _, d := range all {
|
|
have[d] = true
|
|
}
|
|
chosen := map[string]bool{}
|
|
todo := []string{dir}
|
|
if !have[dir] {
|
|
// A directory with no go.mod of its own (a module whose Go code lies in go/, a contracts module with a go.mod per
|
|
// seat) is every go.mod under it; the tree is already narrowed to the module's paths (A-447a).
|
|
todo = nil
|
|
for _, d := range all {
|
|
if dir == "." || strings.HasPrefix(d, dir+"/") {
|
|
todo = append(todo, d)
|
|
}
|
|
}
|
|
}
|
|
for len(todo) > 0 {
|
|
d := todo[0]
|
|
todo = todo[1:]
|
|
if chosen[d] || !have[d] {
|
|
continue
|
|
}
|
|
chosen[d] = true
|
|
file := filepath.Join(tree, filepath.FromSlash(d), "go.mod")
|
|
body, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := modfile.Parse(file, body, nil) // not ParseLax: that one ignores the replaces
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", file, err)
|
|
}
|
|
for _, r := range f.Replace {
|
|
if !modfile.IsDirectoryPath(r.New.Path) {
|
|
continue
|
|
}
|
|
target := filepath.ToSlash(filepath.Clean(filepath.Join(filepath.FromSlash(d), filepath.FromSlash(r.New.Path))))
|
|
todo = append(todo, target)
|
|
}
|
|
}
|
|
out := []string{}
|
|
for _, d := range all {
|
|
if chosen[d] {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return nil, fmt.Errorf("the module directory %q holds no Go module (no go.mod in it or under it)", dir)
|
|
}
|
|
return out, nil
|
|
}
|