mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go/fetch.go

package main

// The fetch step (novox/hq ADR 0405 §5 §6, ADR 0466 §4 §5, ADR 0469 §2; issue 1485): a commit of a repository of
// the mesh's forge, fetched over SSH as the module's own account into a workspace of its own, and the Go modules every
// go.mod in it names, downloaded into the module's cache and verified against go.sum.

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/forgekey"
)

var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)

// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
const ForgeAlias = "mesh-forge"

var (
	repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
	workspaceOK  = regexp.MustCompile(`^[a-z0-9]+$`)
)

// FetchResult is what a passed fetch step says: the workspace it made, the commit in it and the directories whose
// Go modules it downloaded.
type FetchResult struct {
	Workspace string   `json:"workspace"`
	Commit    string   `json:"commit"`
	Modules   []string `json:"modules"`
	// The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the
	// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
	// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
	// files, and the workspace is left at the trunk commit.
	Base      string   `json:"base,omitempty"`
	Merge     string   `json:"merge,omitempty"`
	Conflicts []string `json:"conflicts,omitempty"`
	Changed   []string `json:"changed,omitempty"`
	// Programs are the changed files that are compiled programs (issue 1577); a merge answers it, a pointer to an empty list when there are none, absent for a fetch that made no merge.
	Programs *[]buildstep.Program            `json:"programs,omitempty"`
	Trees    map[string]buildstep.ModuleTree `json:"trees,omitempty"`
	// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
	// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
	Beside []Sibling `json:"beside,omitempty"`
	// Packages are, of each directory whose go.mod requires an asked mesh package, the version it is built against
	// (versions.go; novox/hq ADR 0573 §2): the asked, or go.mod's when it names a newer one, said.
	Packages []TakenVersion `json:"packages,omitempty"`
}

// fetchTool is the fetch step tool.
func (m *Module) fetchTool() StepTool {
	return StepTool{
		Name: "build_go_fetch",
		Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
			"workspace of its own, and download and verify the Go modules every go.mod in it names. Answers a step id at " +
			"once; the step ends on its key in the steps state with the workspace's id.",
		Input: map[string]any{
			"type": "object",
			"properties": map[string]any{
				"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
				"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
				"ref":        map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
				"base":       map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
				"dir":        buildstep.FetchRequestProperties()["dir"],
				"paths":      buildstep.FetchRequestProperties()["paths"],
				"packages":   buildstep.FetchRequestProperties()["packages"],
				"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
					"items": map[string]any{"type": "object", "properties": map[string]any{
						"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
						"commit":     map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
					}, "required": []string{"repository", "commit"}}},
			},
			"required": []string{"repository", "commit"},
		},
		Long: true,
		Run:  m.fetch,
	}
}

// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466
// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it
// (issue 1839).
func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
	// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
	// named paths, the workspace is narrowed to them (buildstep.Narrow).
	req, err := buildstep.ParseFetchRequest(args)
	if err != nil {
		return nil, err
	}
	siblings, err := siblingsArg(args["beside"])
	if err != nil {
		return nil, err
	}
	m.CleanWorkspaces(m.now())
	id := buildstep.NewID()
	done := m.spaces().Hold(id)
	defer func() {
		done()
		if err != nil {
			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
				m.Say("the failed fetch's workspace %s: %v", id, rerr)
			}
		}
	}()
	return m.fetchInto(ctx, req, id, siblings)
}

func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id string, siblings []siblingArg) (any, error) {
	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
	var err error
	root := filepath.Join(m.Work, id)
	tree := filepath.Join(root, "tree")
	if err := os.MkdirAll(tree, 0o700); err != nil {
		return nil, err
	}
	remote := m.remote(repository)
	if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.fetchOptions(ctx, repository, commit, ref)...); err != nil {
		return nil, err
	}
	res := FetchResult{Workspace: id, Commit: commit}
	if base != "" {
		if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
			return nil, err
		}
	}
	if err := m.dissociate(ctx, tree); err != nil {
		return nil, err
	}
	if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
		return nil, err
	}
	if (base == "" || res.Merge != "") && len(req.Paths) > 0 {
		if err := buildstep.Narrow(tree, req); err != nil {
			return nil, err
		}
		// Said beside the tree, so the steps that list the tree's go.mod files leave out those narrowed away.
		if err := os.WriteFile(filepath.Join(root, narrowedMark), nil, 0o600); err != nil {
			return nil, err
		}
	}
	dirs, err := m.goModDirs(ctx, tree)
	if err != nil {
		return nil, err
	}
	res.Modules = dirs
	if base != "" && res.Merge == "" {
		return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
	}
	// The directories download side by side, up to the bound (ADR 0536 §2); the first failure in directory order is the
	// step's. Resolving a pin places a repository beside the workspace, which two directories may share: one at a time.
	var pins sync.Mutex
	errs := make([]error, len(dirs))
	taken := make([][]TakenVersion, len(dirs))
	slots := make(chan struct{}, m.parallel())
	var wg sync.WaitGroup
	for i, d := range dirs {
		if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
			continue // a module that vendors resolves nothing
		}
		wg.Add(1)
		slots <- struct{}{}
		go func() {
			defer wg.Done()
			defer func() { <-slots }()
			// The asked versions of the mesh's packages go into the modfile before anything is downloaded (versions.go).
			if taken[i], errs[i] = m.takeVersions(ctx, root, filepath.Join(tree, d), d, req.Packages, &pins); errs[i] != nil {
				return
			}
			errs[i] = m.downloadModules(ctx, root, filepath.Join(tree, d), d, &pins)
		}()
	}
	wg.Wait()
	for _, err := range errs {
		if err != nil {
			return nil, err
		}
	}
	for _, t := range taken {
		res.Packages = append(res.Packages, t...)
	}
	return res, nil
}

// downloadModules downloads and verifies the Go modules of the directory dir (d in the tree). A pin of a forge module
// that Go cannot fetch is looked for among the forge's pull request heads and handed to Go by a modfile (pins.go); a
// failure that names no such pin is the first failure.
func (m *Module) downloadModules(ctx context.Context, root, dir, d string, pins *sync.Mutex) error {
	for {
		failure := m.downloadAndVerify(ctx, dir, d)
		if failure == nil {
			return nil
		}
		pins.Lock()
		progress, err := m.resolvePins(ctx, root, dir, failure.Error())
		pins.Unlock()
		if err != nil {
			return err
		}
		if !progress {
			return failure
		}
	}
}

// verifyLocks hold one mutex per digest of a go.mod and go.sum, so two directories with the same pair verify once.
var verifyLocks sync.Map

// verifiedMark is the file naming a pass of go mod verify for a go.mod and go.sum pair, "" when a pass is not kept:
// the module cache is only read by steps that are shut off with it read-only (ADR 0448 §3), so the pass holds only then.
// A directory with a temporary modfile (an asked version, a pin on a head) verifies the modfile and the sums beside it,
// which a placed repository's path makes the workspace's own.
func (m *Module) verifiedMark(dir string) string {
	if m.Cache == "" || m.Bubblewrap == nil {
		return ""
	}
	from := dir
	if modfile, ok := m.pinModfile(dir); ok {
		from = filepath.Dir(modfile)
	}
	h := sha256.New()
	for _, name := range []string{"go.mod", "go.sum"} {
		raw, _ := os.ReadFile(filepath.Join(from, name))
		fmt.Fprintf(h, "%s %d\n", name, len(raw))
		h.Write(raw)
	}
	return filepath.Join(m.Cache, "verified", hex.EncodeToString(h.Sum(nil)))
}

func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error {
	for _, verb := range []string{"download", "verify"} {
		mark := ""
		if verb == "verify" {
			if mark = m.verifiedMark(dir); mark != "" {
				mu, _ := verifyLocks.LoadOrStore(mark, &sync.Mutex{})
				mu.(*sync.Mutex).Lock()
				defer mu.(*sync.Mutex).Unlock()
				if _, err := os.Stat(mark); err == nil {
					continue // this go.mod and go.sum passed before
				}
			}
		}
		cmd, env := []string{"-C", dir, "mod", verb}, m.goEnvFor(dir)
		if modfile, ok := m.pinModfile(dir); ok && verb == "verify" {
			// go mod verify takes no -modfile: it is run on the modfile itself, which names the placed repositories.
			cmd, env = []string{"-C", filepath.Dir(modfile), "mod", verb}, m.goEnv()
		}
		r := m.Run(ctx, env, "go", cmd...)
		if r.Status != 0 || r.Err != "" {
			return fmt.Errorf("go mod %s in %s: %s", verb, d, causeLine(r.Err+"\n"+r.Stdout+"\n"+r.Stderr))
		}
		if mark != "" {
			// Said after the pass; a mark that cannot be written only means the next fetch verifies again.
			if os.MkdirAll(filepath.Dir(mark), 0o700) == nil {
				os.WriteFile(mark, nil, 0o600)
			}
		}
	}
	return nil
}

// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
	r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod")
	if r.Status != 0 || r.Err != "" {
		return nil, fmt.Errorf("git ls-files: %s", forgekey.Said(r.Err+"\n"+r.Stderr))
	}
	dirs := []string{}
	for _, l := range strings.Split(r.Stdout, "\n") {
		if l = strings.TrimSpace(l); l != "" && !inTestdata(l) {
			if narrowed(tree) {
				if _, err := os.Stat(filepath.Join(tree, filepath.FromSlash(l))); err != nil {
					continue // outside a narrowed workspace's paths (buildstep.Narrow): no module of this build
				}
			}
			dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
		}
	}
	sort.Strings(dirs)
	return dirs, nil
}

// inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules
// of the tree (issue 1810).
func inTestdata(path string) bool {
	for _, seg := range strings.Split(filepath.ToSlash(path), "/") {
		if seg == "testdata" {
			return true
		}
	}
	return false
}

// goEnv is what a go command of a step is given beyond the base environment: the installed go alone (GOTOOLCHAIN=local, ADR 0477 §1: no toolchain is ever downloaded),
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
// that vendors, see goEnvFor.
func (m *Module) goEnv() []string {
	modcache := filepath.Join(m.Cache, "gomod")
	if m.GoModCache != "" {
		modcache = m.GoModCache
	}
	return append([]string{
		"GOTOOLCHAIN=local",
		"GOFLAGS=-mod=mod",
		"GOMODCACHE=" + modcache,
		"GOCACHE=" + filepath.Join(m.Cache, "gobuild"),
		"GOPRIVATE=git.novox.be",
		"GIT_TERMINAL_PROMPT=0",
		"GIT_SSH_COMMAND=ssh -o BatchMode=yes",
	}, m.threadEnv()...)
}

// goEnvFor is goEnv for a command run in dir: a directory that vendors (vendor/modules.txt) is read from its vendor
// directory, as fetch downloaded nothing for it, and every other by its go.mod and go.sum.
func (m *Module) goEnvFor(dir string) []string {
	env := m.goEnv()
	if _, err := os.Stat(filepath.Join(dir, "vendor", "modules.txt")); err == nil {
		env = append(env, "GOFLAGS=-mod=vendor") // the last value of a name is the one used
	} else if modfile, ok := m.pinModfile(dir); ok {
		env = append(env, "GOFLAGS=-mod=mod -modfile="+modfile) // a pin on a pull request's head (pins.go)
	}
	return env
}

// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
// net for a caller that never released one (issue 1839, ADR 0509).
func (m *Module) CleanWorkspaces(now time.Time) {
	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
}

// clock is the time now: Now when a test set it, else the wall clock.
func (m *Module) clock() time.Time {
	if m.Now != nil {
		return m.Now()
	}
	return time.Now()
}
func (m *Module) now() time.Time { return m.clock() }

// causeLine is the first line of a go command's output that says something other than Go's own progress: `go:
// downloading ...` comes first when Go fetches a module, and `# <package>` heads a compile error of go build or go vet
// (issue 1703: the reason said only the package), and the cause of a failure on a later line.
func causeLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "go: downloading ") && !strings.HasPrefix(l, "# ") {
			return l
		}
	}
	return firstLine(s)
}

func firstLine(s string) string {
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			return l
		}
	}
	return ""
}

// narrowedMark is the file beside a workspace's tree saying the fetch narrowed it to a module's named paths.
const narrowedMark = "narrowed"

// narrowed says whether the fetch narrowed the workspace whose tree this is.
func narrowed(tree string) bool {
	_, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark))
	return err == nil
}