jochen
4d1b81b6cb
A merge to the controller's repository moved the route proxy and the build seat's holder whatever it changed, because nothing said which files their builds read. A build of a trunk commit now says its build source per repository: a Go program's import closure, an archive's directory, an image's recipe and the package it names in the new 'compiles' field. That image is built from its build source alone, so a recipe reading past it fails by name, and its fingerprint is over what it was handed.
486 lines
15 KiB
Go
486 lines
15 KiB
Go
package builder
|
|
|
|
import (
|
|
"bufio"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"go/parser"
|
|
"go/token"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// A Go program's build source (novox/hq ADR 0267 rule 1): the files it is built from, derived from its
|
|
// import closure rather than listed by hand, because a list drifts from the imports it describes and a
|
|
// path missing from it is a real change missed — worse than a needless rebuild.
|
|
//
|
|
// **The closure read here is never narrower than `go list -deps`.** Every .go file of a package that is
|
|
// not a test is read, whatever its build constraint, so the imports are the union over every system and
|
|
// tag; a directory is held whole (but for its tests), so a file added to a package is in it; an embed is
|
|
// held by the directory its pattern starts in, everything below it. Read with the standard library's
|
|
// parser and no toolchain: the build machine carries none, and a closure that needed the network to
|
|
// read would be one a build could not say offline.
|
|
//
|
|
// A build source is a list of entries, relative to the root the build sees:
|
|
//
|
|
// dir/ a Go package's directory: every file directly in it but its tests (`*_test.go`)
|
|
// dir/** everything below a directory (an embed)
|
|
// ** the whole tree
|
|
// file one file
|
|
//
|
|
// The root package's directory is `./`.
|
|
|
|
// GoPackageDirEntry is the entry for a Go package's directory.
|
|
func goPackageDirEntry(dir string) string {
|
|
if dir == "" || dir == "." {
|
|
return "./"
|
|
}
|
|
return dir + "/"
|
|
}
|
|
|
|
// SourceHolds is whether a changed file — a path relative to the root the build source was read in — is
|
|
// in that build source.
|
|
func SourceHolds(entries []string, file string) bool {
|
|
file = strings.TrimPrefix(path.Clean("/"+strings.TrimSpace(file)), "/")
|
|
for _, e := range entries {
|
|
switch {
|
|
case e == "**":
|
|
return true
|
|
case strings.HasSuffix(e, "/**"):
|
|
dir := strings.TrimSuffix(e, "/**")
|
|
if dir == "." || dir == "" || file == dir || strings.HasPrefix(file, dir+"/") {
|
|
return true
|
|
}
|
|
case strings.HasSuffix(e, "/"):
|
|
dir := strings.TrimSuffix(e, "/")
|
|
parent := path.Dir(file)
|
|
if (dir == "." && parent == ".") || parent == dir {
|
|
if !strings.HasSuffix(file, "_test.go") {
|
|
return true
|
|
}
|
|
}
|
|
case e == file:
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GoBuildSource is the build source of the Go program whose main package is pkg, a directory relative to
|
|
// root: the directories of every package of its import closure inside root, the embeds those packages
|
|
// name, its module's go.mod, go.sum and vendor/modules.txt, and a go.work wherever one would be read. An
|
|
// entry for a file that does not exist is kept: creating it is a change to the build.
|
|
//
|
|
// Refused — so the build source is not narrowed, and nothing is missed — when the closure cannot be told
|
|
// from the files: no go.mod holds the package, a go.work is present, a local replace leaves root, a file
|
|
// does not parse, or a cgo preamble reaches outside its directory.
|
|
func GoBuildSource(root, pkg string) ([]string, error) {
|
|
root, err := filepath.Abs(root)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rel := path.Clean(strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(pkg)), "/"))
|
|
if rel == ".." || strings.HasPrefix(rel, "../") {
|
|
return nil, fmt.Errorf("the package %q leaves the tree it is built from", pkg)
|
|
}
|
|
if info, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil || !info.IsDir() {
|
|
return nil, fmt.Errorf("%q is not a directory of the tree it is built from", pkg)
|
|
}
|
|
// The module holding the package: the nearest go.mod at or above it, within root.
|
|
modRoot := ""
|
|
for dir := rel; ; dir = path.Dir(dir) {
|
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.mod")); err == nil {
|
|
modRoot = dir
|
|
break
|
|
}
|
|
if dir == "." {
|
|
break
|
|
}
|
|
}
|
|
if modRoot == "" {
|
|
return nil, fmt.Errorf("no go.mod holds %q within the tree it is built from", pkg)
|
|
}
|
|
entries := map[string]bool{}
|
|
file := func(dir, name string) {
|
|
entries[strings.TrimPrefix(path.Join(dir, name), "./")] = true
|
|
}
|
|
file(modRoot, "go.mod")
|
|
file(modRoot, "go.sum")
|
|
file(modRoot, "vendor/modules.txt")
|
|
// A workspace changes how every import resolves; one present is not read past, one created later is a
|
|
// change to the build.
|
|
for dir := modRoot; ; dir = path.Dir(dir) {
|
|
file(dir, "go.work")
|
|
file(dir, "go.work.sum")
|
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.work")); err == nil {
|
|
return nil, fmt.Errorf("%s holds a go.work, and a workspace's imports are not read here", path.Join(dir, "go.work"))
|
|
}
|
|
if dir == "." {
|
|
break
|
|
}
|
|
}
|
|
modPath, replaces, err := readGoMod(filepath.Join(root, filepath.FromSlash(modRoot), "go.mod"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
vendored := false
|
|
if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(modRoot), "vendor", "modules.txt")); err == nil {
|
|
vendored = true
|
|
}
|
|
// resolve is the directories, relative to root, an import may be read from: none for the standard
|
|
// library and for a module outside the tree, which go.mod and go.sum pin. A vendored module replaced
|
|
// by a local directory is both — vendor/ under -mod=vendor, the directory under -mod=mod — and both
|
|
// are held, so neither way of building it is missed.
|
|
resolve := func(importPath string) ([]string, error) {
|
|
within := func(prefix, dir string) (string, bool) {
|
|
if importPath == prefix {
|
|
return dir, true
|
|
}
|
|
if rest, ok := strings.CutPrefix(importPath, prefix+"/"); ok {
|
|
return path.Join(dir, rest), true
|
|
}
|
|
return "", false
|
|
}
|
|
if dir, ok := within(modPath, modRoot); ok {
|
|
return []string{dir}, nil
|
|
}
|
|
var dirs []string
|
|
for _, r := range replaces {
|
|
if sub, ok := within(r.from, ""); ok {
|
|
target := path.Clean(path.Join(modRoot, r.to))
|
|
if target == ".." || strings.HasPrefix(target, "../") {
|
|
return nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", r.from, r.to)
|
|
}
|
|
dirs = append(dirs, path.Join(target, sub))
|
|
// Its go.mod states what it requires, read when it is built from there.
|
|
entries[path.Join(target, "go.mod")] = true
|
|
break
|
|
}
|
|
}
|
|
first, _, _ := strings.Cut(importPath, "/")
|
|
if len(dirs) == 0 && !strings.Contains(first, ".") {
|
|
return nil, nil // the standard library
|
|
}
|
|
if vendored {
|
|
dirs = append(dirs, path.Join(modRoot, "vendor", importPath))
|
|
}
|
|
return dirs, nil
|
|
}
|
|
|
|
seen := map[string]bool{}
|
|
queue := []string{rel}
|
|
for len(queue) > 0 {
|
|
dir := queue[0]
|
|
queue = queue[1:]
|
|
if seen[dir] {
|
|
continue
|
|
}
|
|
seen[dir] = true
|
|
entries[goPackageDirEntry(dir)] = true
|
|
listing, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir)))
|
|
if err != nil {
|
|
// A package that is not there fails the build that imports it; its directory is held, so
|
|
// adding it is a change.
|
|
continue
|
|
}
|
|
for _, f := range listing {
|
|
name := f.Name()
|
|
if f.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") {
|
|
continue
|
|
}
|
|
full := filepath.Join(root, filepath.FromSlash(dir), name)
|
|
imports, embeds, err := goFileReads(full)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err)
|
|
}
|
|
for _, ip := range imports {
|
|
targets, err := resolve(ip)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, target := range targets {
|
|
if !seen[target] {
|
|
queue = append(queue, target)
|
|
}
|
|
}
|
|
}
|
|
for _, e := range embeds {
|
|
entries[path.Join(dir, e)+"/**"] = true
|
|
if !strings.ContainsAny(e, "*?[\\") {
|
|
entries[path.Join(dir, e)] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(entries))
|
|
for e := range entries {
|
|
out = append(out, e)
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
// goReplace is one local replacement in go.mod: an import path read from a directory.
|
|
type goReplace struct{ from, to string }
|
|
|
|
// readGoMod is a go.mod's module path and its replacements by a local directory. A replacement by another
|
|
// module version is resolved by go.sum, which the build source holds.
|
|
func readGoMod(file string) (string, []goReplace, error) {
|
|
f, err := os.Open(file)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
defer f.Close()
|
|
var module string
|
|
var replaces []goReplace
|
|
inReplace := false
|
|
scanner := bufio.NewScanner(f)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if i := strings.Index(line, "//"); i >= 0 {
|
|
line = line[:i]
|
|
}
|
|
line = strings.TrimSpace(line)
|
|
switch {
|
|
case line == "":
|
|
continue
|
|
case inReplace && line == ")":
|
|
inReplace = false
|
|
continue
|
|
case strings.HasPrefix(line, "module "):
|
|
module = unquoteGoMod(strings.TrimSpace(strings.TrimPrefix(line, "module")))
|
|
continue
|
|
case line == "replace (":
|
|
inReplace = true
|
|
continue
|
|
case strings.HasPrefix(line, "replace "):
|
|
line = strings.TrimSpace(strings.TrimPrefix(line, "replace"))
|
|
case !inReplace:
|
|
continue
|
|
}
|
|
left, right, found := strings.Cut(line, "=>")
|
|
if !found {
|
|
continue
|
|
}
|
|
from := strings.Fields(left)
|
|
to := strings.Fields(right)
|
|
if len(from) == 0 || len(to) == 0 {
|
|
continue
|
|
}
|
|
target := unquoteGoMod(to[0])
|
|
// A local replacement is a path: ./, ../ or absolute. Anything else names a module version.
|
|
if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || target == "." || target == ".." {
|
|
replaces = append(replaces, goReplace{from: unquoteGoMod(from[0]), to: target})
|
|
} else if strings.HasPrefix(target, "/") {
|
|
return "", nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", from[0], target)
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return "", nil, err
|
|
}
|
|
if module == "" {
|
|
return "", nil, fmt.Errorf("%s names no module", file)
|
|
}
|
|
// The longest replacement first, so a replaced sub-path wins over its parent.
|
|
sort.SliceStable(replaces, func(i, j int) bool { return len(replaces[i].from) > len(replaces[j].from) })
|
|
return module, replaces, nil
|
|
}
|
|
|
|
func unquoteGoMod(s string) string {
|
|
if u, err := strconv.Unquote(s); err == nil {
|
|
return u
|
|
}
|
|
return s
|
|
}
|
|
|
|
// goFileReads is what one Go file makes its build read: the packages it imports, and the directories its
|
|
// //go:embed patterns start in, relative to its own directory ("." for the directory itself).
|
|
//
|
|
// **A cgo preamble reaching outside its directory is refused**: a header included by a relative path, or
|
|
// a flag naming ${SRCDIR}/.., is a file of the build no import names. Inside the directory it is held
|
|
// already.
|
|
func goFileReads(file string) (imports, embeds []string, err error) {
|
|
src, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
fset := token.NewFileSet()
|
|
parsed, err := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.ParseComments)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for _, spec := range parsed.Imports {
|
|
ip, err := strconv.Unquote(spec.Path.Value)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if ip == "C" {
|
|
// The preamble is the comment before the import; only its #include and #cgo lines read files.
|
|
for _, cg := range parsed.Comments {
|
|
if cg.End() > spec.Pos() {
|
|
continue
|
|
}
|
|
for _, line := range strings.Split(cg.Text(), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if (strings.HasPrefix(line, "#include") || strings.HasPrefix(line, "#cgo")) && strings.Contains(line, "..") {
|
|
return nil, nil, errors.New("its cgo preamble names a path outside its directory: " + line)
|
|
}
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
imports = append(imports, ip)
|
|
}
|
|
// //go:embed directives may stand anywhere in the file, so the whole text is read for them.
|
|
scanner := bufio.NewScanner(strings.NewReader(string(src)))
|
|
scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
rest, ok := strings.CutPrefix(line, "//go:embed")
|
|
if !ok || (rest != "" && rest[0] != ' ' && rest[0] != '\t') {
|
|
continue
|
|
}
|
|
patterns, err := embedPatterns(rest)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
for _, p := range patterns {
|
|
embeds = append(embeds, embedRoot(p))
|
|
}
|
|
}
|
|
return imports, embeds, scanner.Err()
|
|
}
|
|
|
|
// embedPatterns splits a //go:embed line's patterns: separated by spaces, each possibly quoted.
|
|
func embedPatterns(s string) ([]string, error) {
|
|
var out []string
|
|
s = strings.TrimSpace(s)
|
|
for s != "" {
|
|
var p string
|
|
switch s[0] {
|
|
case '"', '`':
|
|
q, err := strconv.QuotedPrefix(s)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("an embed pattern does not parse: %w", err)
|
|
}
|
|
if p, err = strconv.Unquote(q); err != nil {
|
|
return nil, err
|
|
}
|
|
s = s[len(q):]
|
|
default:
|
|
end := strings.IndexAny(s, " \t")
|
|
if end < 0 {
|
|
end = len(s)
|
|
}
|
|
p, s = s[:end], s[end:]
|
|
}
|
|
out = append(out, p)
|
|
s = strings.TrimSpace(s)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// embedRoot is the directory an embed pattern starts in, relative to the package: its leading elements
|
|
// without a wildcard. A pattern naming a file or a directory outright is held as itself.
|
|
func embedRoot(pattern string) string {
|
|
pattern = strings.TrimPrefix(pattern, "all:")
|
|
var kept []string
|
|
for _, el := range strings.Split(pattern, "/") {
|
|
if strings.ContainsAny(el, "*?[\\") {
|
|
break
|
|
}
|
|
kept = append(kept, el)
|
|
}
|
|
if len(kept) == 0 {
|
|
return "."
|
|
}
|
|
return path.Clean(strings.Join(kept, "/"))
|
|
}
|
|
|
|
// narrowTree copies into dst the files of src a build source holds, and nothing else — never `.git` — and
|
|
// returns a fingerprint of what it copied: each file's path, mode and content, hashed in path order. **A
|
|
// build handed only its build source cannot read past it** (novox/hq ADR 0267 rule 3): a recipe that
|
|
// copies a file outside it fails, naming the file, where it would have built and been missed.
|
|
func narrowTree(src, dst string, entries []string) (string, error) {
|
|
if err := os.RemoveAll(dst); err != nil {
|
|
return "", err
|
|
}
|
|
if err := os.MkdirAll(dst, 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
var lines []string
|
|
err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(src, p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel = filepath.ToSlash(rel)
|
|
if d.IsDir() {
|
|
if d.Name() == ".git" {
|
|
return filepath.SkipDir
|
|
}
|
|
return nil
|
|
}
|
|
if !SourceHolds(entries, rel) {
|
|
return nil
|
|
}
|
|
out := filepath.Join(dst, filepath.FromSlash(rel))
|
|
if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
|
|
return err
|
|
}
|
|
info, err := d.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&fs.ModeSymlink != 0 {
|
|
// A link in the repository is copied as the link it is, and what it names said in the
|
|
// fingerprint.
|
|
target, err := os.Readlink(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
lines = append(lines, fmt.Sprintf("%s link %s", rel, target))
|
|
return os.Symlink(target, out)
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return nil
|
|
}
|
|
in, err := os.Open(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer in.Close()
|
|
w, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sum := sha256.New()
|
|
if _, err := io.Copy(io.MultiWriter(w, sum), in); err != nil {
|
|
w.Close()
|
|
return err
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
return err
|
|
}
|
|
lines = append(lines, fmt.Sprintf("%s %o %s", rel, info.Mode().Perm()&0o111, hex.EncodeToString(sum.Sum(nil))))
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sort.Strings(lines)
|
|
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
|
|
return "narrow:" + hex.EncodeToString(sum[:]), nil
|
|
}
|