mesh-controller/internal/builder/gosource.go

package builder

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"go/parser"
	"go/token"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
)

// A Go program's build source (novox/hq ADR 0267 rule 1): the files it is built from, derived from its
// import closure rather than listed by hand, because a list drifts from the imports it describes and a
// path missing from it is a real change missed — worse than a needless rebuild.
//
// **The closure read here is never narrower than `go list -deps`.** Every .go file of a package that is
// not a test is read, whatever its build constraint, so the imports are the union over every system and
// tag; a directory is held whole (but for its tests), so a file added to a package is in it; an embed is
// held by the directory its pattern starts in, everything below it. Read with the standard library's
// parser and no toolchain: the build machine carries none, and a closure that needed the network to
// read would be one a build could not say offline.
//
// A build source is a list of entries, relative to the root the build sees:
//
//	dir/      a Go package's directory: every file directly in it but its tests (`*_test.go`)
//	dir/**    everything below a directory (an embed)
//	**        the whole tree
//	file      one file
//
// The root package's directory is `./`.

// GoPackageDirEntry is the entry for a Go package's directory.
func goPackageDirEntry(dir string) string {
	if dir == "" || dir == "." {
		return "./"
	}
	return dir + "/"
}

// SourceHolds is whether a changed file — a path relative to the root the build source was read in — is
// in that build source.
func SourceHolds(entries []string, file string) bool {
	file = strings.TrimPrefix(path.Clean("/"+strings.TrimSpace(file)), "/")
	for _, e := range entries {
		switch {
		case e == "**":
			return true
		case strings.HasSuffix(e, "/**"):
			dir := strings.TrimSuffix(e, "/**")
			if dir == "." || dir == "" || file == dir || strings.HasPrefix(file, dir+"/") {
				return true
			}
		case strings.HasSuffix(e, "/"):
			dir := strings.TrimSuffix(e, "/")
			parent := path.Dir(file)
			if (dir == "." && parent == ".") || parent == dir {
				if !strings.HasSuffix(file, "_test.go") {
					return true
				}
			}
		case e == file:
			return true
		}
	}
	return false
}

// GoBuildSource is the build source of the Go program whose main package is pkg, a directory relative to
// root: the directories of every package of its import closure inside root, the embeds those packages
// name, its module's go.mod, go.sum and vendor/modules.txt, and a go.work wherever one would be read. An
// entry for a file that does not exist is kept: creating it is a change to the build.
//
// Refused — so the build source is not narrowed, and nothing is missed — when the closure cannot be told
// from the files: no go.mod holds the package, a go.work is present, a local replace leaves root, a file
// does not parse, or a cgo preamble reaches outside its directory.
func GoBuildSource(root, pkg string) ([]string, error) {
	root, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}
	rel := path.Clean(strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(pkg)), "/"))
	if rel == ".." || strings.HasPrefix(rel, "../") {
		return nil, fmt.Errorf("the package %q leaves the tree it is built from", pkg)
	}
	if info, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil || !info.IsDir() {
		return nil, fmt.Errorf("%q is not a directory of the tree it is built from", pkg)
	}
	// The module holding the package: the nearest go.mod at or above it, within root.
	modRoot := ""
	for dir := rel; ; dir = path.Dir(dir) {
		if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.mod")); err == nil {
			modRoot = dir
			break
		}
		if dir == "." {
			break
		}
	}
	if modRoot == "" {
		return nil, fmt.Errorf("no go.mod holds %q within the tree it is built from", pkg)
	}
	entries := map[string]bool{}
	file := func(dir, name string) {
		entries[strings.TrimPrefix(path.Join(dir, name), "./")] = true
	}
	file(modRoot, "go.mod")
	file(modRoot, "go.sum")
	file(modRoot, "vendor/modules.txt")
	// A workspace changes how every import resolves; one present is not read past, one created later is a
	// change to the build.
	for dir := modRoot; ; dir = path.Dir(dir) {
		file(dir, "go.work")
		file(dir, "go.work.sum")
		if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.work")); err == nil {
			return nil, fmt.Errorf("%s holds a go.work, and a workspace's imports are not read here", path.Join(dir, "go.work"))
		}
		if dir == "." {
			break
		}
	}
	modPath, replaces, err := readGoMod(filepath.Join(root, filepath.FromSlash(modRoot), "go.mod"))
	if err != nil {
		return nil, err
	}
	vendored := false
	if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(modRoot), "vendor", "modules.txt")); err == nil {
		vendored = true
	}
	// resolve is the directories, relative to root, an import may be read from: none for the standard
	// library and for a module outside the tree, which go.mod and go.sum pin. A vendored module replaced
	// by a local directory is both — vendor/ under -mod=vendor, the directory under -mod=mod — and both
	// are held, so neither way of building it is missed.
	resolve := func(importPath string) ([]string, error) {
		within := func(prefix, dir string) (string, bool) {
			if importPath == prefix {
				return dir, true
			}
			if rest, ok := strings.CutPrefix(importPath, prefix+"/"); ok {
				return path.Join(dir, rest), true
			}
			return "", false
		}
		if dir, ok := within(modPath, modRoot); ok {
			return []string{dir}, nil
		}
		var dirs []string
		for _, r := range replaces {
			if sub, ok := within(r.from, ""); ok {
				target := path.Clean(path.Join(modRoot, r.to))
				if target == ".." || strings.HasPrefix(target, "../") {
					return nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", r.from, r.to)
				}
				dirs = append(dirs, path.Join(target, sub))
				// Its go.mod states what it requires, read when it is built from there.
				entries[path.Join(target, "go.mod")] = true
				break
			}
		}
		first, _, _ := strings.Cut(importPath, "/")
		if len(dirs) == 0 && !strings.Contains(first, ".") {
			return nil, nil // the standard library
		}
		if vendored {
			dirs = append(dirs, path.Join(modRoot, "vendor", importPath))
		}
		return dirs, nil
	}

	seen := map[string]bool{}
	queue := []string{rel}
	for len(queue) > 0 {
		dir := queue[0]
		queue = queue[1:]
		if seen[dir] {
			continue
		}
		seen[dir] = true
		entries[goPackageDirEntry(dir)] = true
		// A go.mod made between the module's root and a package moves that package out of the module.
		for up := dir; up != modRoot && up != "." && up != "/" && !strings.HasPrefix(up, "../"); up = path.Dir(up) {
			file(up, "go.mod")
		}
		listing, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir)))
		if err != nil {
			// A package that is not there fails the build that imports it; its directory is held, so
			// adding it is a change.
			continue
		}
		for _, f := range listing {
			name := f.Name()
			// **C and assembly beside Go** may include files from below the package's directory: the
			// directory is held whole, and an include reaching above it is refused.
			if !f.IsDir() && nativeSource(name) {
				entries[strings.TrimPrefix(dir+"/**", "./")] = true
				if dir == "." {
					entries["**"] = true
				}
				if err := includesStayWithin(filepath.Join(root, filepath.FromSlash(dir), name)); err != nil {
					return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err)
				}
				continue
			}
			if f.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
				continue
			}
			full := filepath.Join(root, filepath.FromSlash(dir), name)
			imports, embeds, err := goFileReads(full)
			if err != nil {
				return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err)
			}
			for _, ip := range imports {
				targets, err := resolve(ip)
				if err != nil {
					return nil, err
				}
				for _, target := range targets {
					if !seen[target] {
						queue = append(queue, target)
					}
				}
			}
			for _, e := range embeds {
				entries[path.Join(dir, e)+"/**"] = true
				if !strings.ContainsAny(e, "*?[\\") {
					entries[path.Join(dir, e)] = true
				}
			}
		}
	}
	out := make([]string, 0, len(entries))
	for e := range entries {
		out = append(out, e)
	}
	sort.Strings(out)
	return out, nil
}

// goReplace is one local replacement in go.mod: an import path read from a directory.
type goReplace struct{ from, to string }

// readGoMod is a go.mod's module path and its replacements by a local directory. A replacement by another
// module version is resolved by go.sum, which the build source holds.
func readGoMod(file string) (string, []goReplace, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", nil, err
	}
	defer f.Close()
	var module string
	var replaces []goReplace
	inReplace := false
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if i := strings.Index(line, "//"); i >= 0 {
			line = line[:i]
		}
		line = strings.TrimSpace(line)
		switch {
		case line == "":
			continue
		case inReplace && line == ")":
			inReplace = false
			continue
		case strings.HasPrefix(line, "module "):
			module = unquoteGoMod(strings.TrimSpace(strings.TrimPrefix(line, "module")))
			continue
		case line == "replace (":
			inReplace = true
			continue
		case strings.HasPrefix(line, "replace "):
			line = strings.TrimSpace(strings.TrimPrefix(line, "replace"))
		case !inReplace:
			continue
		}
		left, right, found := strings.Cut(line, "=>")
		if !found {
			continue
		}
		from := strings.Fields(left)
		to := strings.Fields(right)
		if len(from) == 0 || len(to) == 0 {
			continue
		}
		target := unquoteGoMod(to[0])
		// A local replacement is a path: ./, ../ or absolute. Anything else names a module version.
		if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || target == "." || target == ".." {
			replaces = append(replaces, goReplace{from: unquoteGoMod(from[0]), to: target})
		} else if strings.HasPrefix(target, "/") {
			return "", nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", from[0], target)
		}
	}
	if err := scanner.Err(); err != nil {
		return "", nil, err
	}
	if module == "" {
		return "", nil, fmt.Errorf("%s names no module", file)
	}
	// The longest replacement first, so a replaced sub-path wins over its parent.
	sort.SliceStable(replaces, func(i, j int) bool { return len(replaces[i].from) > len(replaces[j].from) })
	return module, replaces, nil
}

func unquoteGoMod(s string) string {
	if u, err := strconv.Unquote(s); err == nil {
		return u
	}
	return s
}

// goFileReads is what one Go file makes its build read: the packages it imports, and the directories its
// //go:embed patterns start in, relative to its own directory ("." for the directory itself).
//
// **A cgo preamble reaching outside its directory is refused**: a header included by a relative path, or
// a flag naming ${SRCDIR}/.., is a file of the build no import names. Inside the directory it is held
// already.
func goFileReads(file string) (imports, embeds []string, err error) {
	src, err := os.ReadFile(file)
	if err != nil {
		return nil, nil, err
	}
	fset := token.NewFileSet()
	parsed, err := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.ParseComments)
	if err != nil {
		return nil, nil, err
	}
	for _, spec := range parsed.Imports {
		ip, err := strconv.Unquote(spec.Path.Value)
		if err != nil {
			return nil, nil, err
		}
		if ip == "C" {
			// The preamble is the comment before the import; only its #include and #cgo lines read files.
			for _, cg := range parsed.Comments {
				if cg.End() > spec.Pos() {
					continue
				}
				for _, line := range strings.Split(cg.Text(), "\n") {
					line = strings.TrimSpace(line)
					if (strings.HasPrefix(line, "#include") || strings.HasPrefix(line, "#cgo")) && strings.Contains(line, "..") {
						return nil, nil, errors.New("its cgo preamble names a path outside its directory: " + line)
					}
				}
			}
			// A cgo file may include from below its directory: the directory is held whole.
			embeds = append(embeds, ".")
			continue
		}
		imports = append(imports, ip)
	}
	// //go:embed directives may stand anywhere in the file, so the whole text is read for them.
	scanner := bufio.NewScanner(strings.NewReader(string(src)))
	scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		rest, ok := strings.CutPrefix(line, "//go:embed")
		if !ok || (rest != "" && rest[0] != ' ' && rest[0] != '\t') {
			continue
		}
		patterns, err := embedPatterns(rest)
		if err != nil {
			return nil, nil, err
		}
		for _, p := range patterns {
			embeds = append(embeds, embedRoot(p))
		}
	}
	return imports, embeds, scanner.Err()
}

// nativeSource is a file the Go command compiles or links beside Go: C, C++, Objective-C, Fortran,
// assembly, their headers and a system object.
func nativeSource(name string) bool {
	switch strings.ToLower(path.Ext(name)) {
	case ".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".m", ".s", ".sx", ".f", ".f90", ".for", ".syso":
		return true
	}
	return false
}

// includesStayWithin refuses a native source whose #include names a path above its directory.
func includesStayWithin(file string) error {
	body, err := os.ReadFile(file)
	if err != nil {
		return err
	}
	for _, line := range strings.Split(string(body), "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#") && strings.Contains(line, "include") && strings.Contains(line, "..") {
			return errors.New("it includes a path outside its directory: " + line)
		}
	}
	return nil
}

// embedPatterns splits a //go:embed line's patterns: separated by spaces, each possibly quoted.
func embedPatterns(s string) ([]string, error) {
	var out []string
	s = strings.TrimSpace(s)
	for s != "" {
		var p string
		switch s[0] {
		case '"', '`':
			q, err := strconv.QuotedPrefix(s)
			if err != nil {
				return nil, fmt.Errorf("an embed pattern does not parse: %w", err)
			}
			if p, err = strconv.Unquote(q); err != nil {
				return nil, err
			}
			s = s[len(q):]
		default:
			end := strings.IndexAny(s, " \t")
			if end < 0 {
				end = len(s)
			}
			p, s = s[:end], s[end:]
		}
		out = append(out, p)
		s = strings.TrimSpace(s)
	}
	return out, nil
}

// embedRoot is the directory an embed pattern starts in, relative to the package: its leading elements
// without a wildcard. A pattern naming a file or a directory outright is held as itself.
func embedRoot(pattern string) string {
	pattern = strings.TrimPrefix(pattern, "all:")
	var kept []string
	for _, el := range strings.Split(pattern, "/") {
		if strings.ContainsAny(el, "*?[\\") {
			break
		}
		kept = append(kept, el)
	}
	if len(kept) == 0 {
		return "."
	}
	return path.Clean(strings.Join(kept, "/"))
}

// narrowTree copies into dst the files of src a build source holds, and nothing else — never `.git` — and
// returns a fingerprint of what it copied: each file's path, mode and content, hashed in path order. **A
// build handed only its build source cannot read past it** (novox/hq ADR 0267 rule 3): a recipe that
// copies a file outside it fails, naming the file, where it would have built and been missed.
func narrowTree(src, dst string, entries []string) (string, error) {
	if err := os.RemoveAll(dst); err != nil {
		return "", err
	}
	if err := os.MkdirAll(dst, 0o755); err != nil {
		return "", err
	}
	var lines []string
	err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(src, p)
		if err != nil {
			return err
		}
		rel = filepath.ToSlash(rel)
		if d.IsDir() {
			if d.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}
		if !SourceHolds(entries, rel) {
			return nil
		}
		out := filepath.Join(dst, filepath.FromSlash(rel))
		if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
			return err
		}
		info, err := d.Info()
		if err != nil {
			return err
		}
		if info.Mode()&fs.ModeSymlink != 0 {
			// A link in the repository is copied as the link it is, and what it names said in the
			// fingerprint.
			target, err := os.Readlink(p)
			if err != nil {
				return err
			}
			lines = append(lines, fmt.Sprintf("%s link %s", rel, target))
			return os.Symlink(target, out)
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		in, err := os.Open(p)
		if err != nil {
			return err
		}
		defer in.Close()
		w, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm())
		if err != nil {
			return err
		}
		sum := sha256.New()
		if _, err := io.Copy(io.MultiWriter(w, sum), in); err != nil {
			w.Close()
			return err
		}
		if err := w.Close(); err != nil {
			return err
		}
		lines = append(lines, fmt.Sprintf("%s %o %s", rel, info.Mode().Perm()&0o111, hex.EncodeToString(sum.Sum(nil))))
		return nil
	})
	if err != nil {
		return "", err
	}
	sort.Strings(lines)
	sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
	return "narrow:" + hex.EncodeToString(sum[:]), nil
}