package builder import ( "bufio" "crypto/sha256" "encoding/hex" "errors" "fmt" "go/parser" "go/token" "io" "io/fs" "os" "path" "path/filepath" "sort" "strconv" "strings" ) // A Go program's build source (novox/hq ADR 0267 rule 1): the files it is built from, derived from its // import closure rather than listed by hand, because a list drifts from the imports it describes and a // path missing from it is a real change missed — worse than a needless rebuild. // // **The closure read here is never narrower than `go list -deps`.** Every .go file of a package that is // not a test is read, whatever its build constraint, so the imports are the union over every system and // tag; a directory is held whole (but for its tests), so a file added to a package is in it; an embed is // held by the directory its pattern starts in, everything below it. Read with the standard library's // parser and no toolchain: the build machine carries none, and a closure that needed the network to // read would be one a build could not say offline. // // A build source is a list of entries, relative to the root the build sees: // // dir/ a Go package's directory: every file directly in it but its tests (`*_test.go`) // dir/** everything below a directory (an embed) // ** the whole tree // file one file // // The root package's directory is `./`. // GoPackageDirEntry is the entry for a Go package's directory. func goPackageDirEntry(dir string) string { if dir == "" || dir == "." { return "./" } return dir + "/" } // SourceHolds is whether a changed file — a path relative to the root the build source was read in — is // in that build source. func SourceHolds(entries []string, file string) bool { file = strings.TrimPrefix(path.Clean("/"+strings.TrimSpace(file)), "/") for _, e := range entries { switch { case e == "**": return true case strings.HasSuffix(e, "/**"): dir := strings.TrimSuffix(e, "/**") if dir == "." || dir == "" || file == dir || strings.HasPrefix(file, dir+"/") { return true } case strings.HasSuffix(e, "/"): dir := strings.TrimSuffix(e, "/") parent := path.Dir(file) if (dir == "." && parent == ".") || parent == dir { if !strings.HasSuffix(file, "_test.go") { return true } } case e == file: return true } } return false } // GoBuildSource is the build source of the Go program whose main package is pkg, a directory relative to // root: the directories of every package of its import closure inside root, the embeds those packages // name, its module's go.mod, go.sum and vendor/modules.txt, and a go.work wherever one would be read. An // entry for a file that does not exist is kept: creating it is a change to the build. // // Refused — so the build source is not narrowed, and nothing is missed — when the closure cannot be told // from the files: no go.mod holds the package, a go.work is present, a local replace leaves root, a file // does not parse, or a cgo preamble reaches outside its directory. func GoBuildSource(root, pkg string) ([]string, error) { root, err := filepath.Abs(root) if err != nil { return nil, err } rel := path.Clean(strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(pkg)), "/")) if rel == ".." || strings.HasPrefix(rel, "../") { return nil, fmt.Errorf("the package %q leaves the tree it is built from", pkg) } if info, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil || !info.IsDir() { return nil, fmt.Errorf("%q is not a directory of the tree it is built from", pkg) } // The module holding the package: the nearest go.mod at or above it, within root. modRoot := "" for dir := rel; ; dir = path.Dir(dir) { if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.mod")); err == nil { modRoot = dir break } if dir == "." { break } } if modRoot == "" { return nil, fmt.Errorf("no go.mod holds %q within the tree it is built from", pkg) } entries := map[string]bool{} file := func(dir, name string) { entries[strings.TrimPrefix(path.Join(dir, name), "./")] = true } file(modRoot, "go.mod") file(modRoot, "go.sum") file(modRoot, "vendor/modules.txt") // A workspace changes how every import resolves; one present is not read past, one created later is a // change to the build. for dir := modRoot; ; dir = path.Dir(dir) { file(dir, "go.work") file(dir, "go.work.sum") if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir), "go.work")); err == nil { return nil, fmt.Errorf("%s holds a go.work, and a workspace's imports are not read here", path.Join(dir, "go.work")) } if dir == "." { break } } modPath, replaces, err := readGoMod(filepath.Join(root, filepath.FromSlash(modRoot), "go.mod")) if err != nil { return nil, err } vendored := false if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(modRoot), "vendor", "modules.txt")); err == nil { vendored = true } // resolve is the directories, relative to root, an import may be read from: none for the standard // library and for a module outside the tree, which go.mod and go.sum pin. A vendored module replaced // by a local directory is both — vendor/ under -mod=vendor, the directory under -mod=mod — and both // are held, so neither way of building it is missed. resolve := func(importPath string) ([]string, error) { within := func(prefix, dir string) (string, bool) { if importPath == prefix { return dir, true } if rest, ok := strings.CutPrefix(importPath, prefix+"/"); ok { return path.Join(dir, rest), true } return "", false } if dir, ok := within(modPath, modRoot); ok { return []string{dir}, nil } var dirs []string for _, r := range replaces { if sub, ok := within(r.from, ""); ok { target := path.Clean(path.Join(modRoot, r.to)) if target == ".." || strings.HasPrefix(target, "../") { return nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", r.from, r.to) } dirs = append(dirs, path.Join(target, sub)) // Its go.mod states what it requires, read when it is built from there. entries[path.Join(target, "go.mod")] = true break } } first, _, _ := strings.Cut(importPath, "/") if len(dirs) == 0 && !strings.Contains(first, ".") { return nil, nil // the standard library } if vendored { dirs = append(dirs, path.Join(modRoot, "vendor", importPath)) } return dirs, nil } seen := map[string]bool{} queue := []string{rel} for len(queue) > 0 { dir := queue[0] queue = queue[1:] if seen[dir] { continue } seen[dir] = true entries[goPackageDirEntry(dir)] = true // A go.mod made between the module's root and a package moves that package out of the module. for up := dir; up != modRoot && up != "." && up != "/" && !strings.HasPrefix(up, "../"); up = path.Dir(up) { file(up, "go.mod") } listing, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(dir))) if err != nil { // A package that is not there fails the build that imports it; its directory is held, so // adding it is a change. continue } for _, f := range listing { name := f.Name() // **C and assembly beside Go** may include files from below the package's directory: the // directory is held whole, and an include reaching above it is refused. if !f.IsDir() && nativeSource(name) { entries[strings.TrimPrefix(dir+"/**", "./")] = true if dir == "." { entries["**"] = true } if err := includesStayWithin(filepath.Join(root, filepath.FromSlash(dir), name)); err != nil { return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err) } continue } if f.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { continue } full := filepath.Join(root, filepath.FromSlash(dir), name) imports, embeds, err := goFileReads(full) if err != nil { return nil, fmt.Errorf("%s: %w", path.Join(dir, name), err) } for _, ip := range imports { targets, err := resolve(ip) if err != nil { return nil, err } for _, target := range targets { if !seen[target] { queue = append(queue, target) } } } for _, e := range embeds { entries[path.Join(dir, e)+"/**"] = true if !strings.ContainsAny(e, "*?[\\") { entries[path.Join(dir, e)] = true } } } } out := make([]string, 0, len(entries)) for e := range entries { out = append(out, e) } sort.Strings(out) return out, nil } // goReplace is one local replacement in go.mod: an import path read from a directory. type goReplace struct{ from, to string } // readGoMod is a go.mod's module path and its replacements by a local directory. A replacement by another // module version is resolved by go.sum, which the build source holds. func readGoMod(file string) (string, []goReplace, error) { f, err := os.Open(file) if err != nil { return "", nil, err } defer f.Close() var module string var replaces []goReplace inReplace := false scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() if i := strings.Index(line, "//"); i >= 0 { line = line[:i] } line = strings.TrimSpace(line) switch { case line == "": continue case inReplace && line == ")": inReplace = false continue case strings.HasPrefix(line, "module "): module = unquoteGoMod(strings.TrimSpace(strings.TrimPrefix(line, "module"))) continue case line == "replace (": inReplace = true continue case strings.HasPrefix(line, "replace "): line = strings.TrimSpace(strings.TrimPrefix(line, "replace")) case !inReplace: continue } left, right, found := strings.Cut(line, "=>") if !found { continue } from := strings.Fields(left) to := strings.Fields(right) if len(from) == 0 || len(to) == 0 { continue } target := unquoteGoMod(to[0]) // A local replacement is a path: ./, ../ or absolute. Anything else names a module version. if strings.HasPrefix(target, "./") || strings.HasPrefix(target, "../") || target == "." || target == ".." { replaces = append(replaces, goReplace{from: unquoteGoMod(from[0]), to: target}) } else if strings.HasPrefix(target, "/") { return "", nil, fmt.Errorf("go.mod replaces %s with %s, outside the tree it is built from", from[0], target) } } if err := scanner.Err(); err != nil { return "", nil, err } if module == "" { return "", nil, fmt.Errorf("%s names no module", file) } // The longest replacement first, so a replaced sub-path wins over its parent. sort.SliceStable(replaces, func(i, j int) bool { return len(replaces[i].from) > len(replaces[j].from) }) return module, replaces, nil } func unquoteGoMod(s string) string { if u, err := strconv.Unquote(s); err == nil { return u } return s } // goFileReads is what one Go file makes its build read: the packages it imports, and the directories its // //go:embed patterns start in, relative to its own directory ("." for the directory itself). // // **A cgo preamble reaching outside its directory is refused**: a header included by a relative path, or // a flag naming ${SRCDIR}/.., is a file of the build no import names. Inside the directory it is held // already. func goFileReads(file string) (imports, embeds []string, err error) { src, err := os.ReadFile(file) if err != nil { return nil, nil, err } fset := token.NewFileSet() parsed, err := parser.ParseFile(fset, file, src, parser.ImportsOnly|parser.ParseComments) if err != nil { return nil, nil, err } for _, spec := range parsed.Imports { ip, err := strconv.Unquote(spec.Path.Value) if err != nil { return nil, nil, err } if ip == "C" { // The preamble is the comment before the import; only its #include and #cgo lines read files. for _, cg := range parsed.Comments { if cg.End() > spec.Pos() { continue } for _, line := range strings.Split(cg.Text(), "\n") { line = strings.TrimSpace(line) if (strings.HasPrefix(line, "#include") || strings.HasPrefix(line, "#cgo")) && strings.Contains(line, "..") { return nil, nil, errors.New("its cgo preamble names a path outside its directory: " + line) } } } continue } imports = append(imports, ip) } // //go:embed directives may stand anywhere in the file, so the whole text is read for them. scanner := bufio.NewScanner(strings.NewReader(string(src))) scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) rest, ok := strings.CutPrefix(line, "//go:embed") if !ok || (rest != "" && rest[0] != ' ' && rest[0] != '\t') { continue } patterns, err := embedPatterns(rest) if err != nil { return nil, nil, err } for _, p := range patterns { embeds = append(embeds, embedRoot(p)) } } return imports, embeds, scanner.Err() } // nativeSource is a file the Go command compiles or links beside Go: C, C++, Objective-C, Fortran, // assembly, their headers and a system object. func nativeSource(name string) bool { switch strings.ToLower(path.Ext(name)) { case ".c", ".h", ".cc", ".cpp", ".cxx", ".hh", ".hpp", ".hxx", ".m", ".s", ".sx", ".f", ".f90", ".for", ".syso": return true } return false } // includesStayWithin refuses a native source whose #include names a path above its directory. func includesStayWithin(file string) error { body, err := os.ReadFile(file) if err != nil { return err } for _, line := range strings.Split(string(body), "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "#") && strings.Contains(line, "include") && strings.Contains(line, "..") { return errors.New("it includes a path outside its directory: " + line) } } return nil } // embedPatterns splits a //go:embed line's patterns: separated by spaces, each possibly quoted. func embedPatterns(s string) ([]string, error) { var out []string s = strings.TrimSpace(s) for s != "" { var p string switch s[0] { case '"', '`': q, err := strconv.QuotedPrefix(s) if err != nil { return nil, fmt.Errorf("an embed pattern does not parse: %w", err) } if p, err = strconv.Unquote(q); err != nil { return nil, err } s = s[len(q):] default: end := strings.IndexAny(s, " \t") if end < 0 { end = len(s) } p, s = s[:end], s[end:] } out = append(out, p) s = strings.TrimSpace(s) } return out, nil } // embedRoot is the directory an embed pattern starts in, relative to the package: its leading elements // without a wildcard. A pattern naming a file or a directory outright is held as itself. func embedRoot(pattern string) string { pattern = strings.TrimPrefix(pattern, "all:") var kept []string for _, el := range strings.Split(pattern, "/") { if strings.ContainsAny(el, "*?[\\") { break } kept = append(kept, el) } if len(kept) == 0 { return "." } return path.Clean(strings.Join(kept, "/")) } // narrowTree copies into dst the files of src a build source holds, and nothing else — never `.git` — and // returns a fingerprint of what it copied: each file's path, mode and content, hashed in path order. **A // build handed only its build source cannot read past it** (novox/hq ADR 0267 rule 3): a recipe that // copies a file outside it fails, naming the file, where it would have built and been missed. func narrowTree(src, dst string, entries []string) (string, error) { if err := os.RemoveAll(dst); err != nil { return "", err } if err := os.MkdirAll(dst, 0o755); err != nil { return "", err } var lines []string err := filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } rel, err := filepath.Rel(src, p) if err != nil { return err } rel = filepath.ToSlash(rel) if d.IsDir() { if d.Name() == ".git" { return filepath.SkipDir } return nil } if !SourceHolds(entries, rel) { return nil } out := filepath.Join(dst, filepath.FromSlash(rel)) if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { return err } info, err := d.Info() if err != nil { return err } if info.Mode()&fs.ModeSymlink != 0 { // A link in the repository is copied as the link it is, and what it names said in the // fingerprint. target, err := os.Readlink(p) if err != nil { return err } lines = append(lines, fmt.Sprintf("%s link %s", rel, target)) return os.Symlink(target, out) } if !info.Mode().IsRegular() { return nil } in, err := os.Open(p) if err != nil { return err } defer in.Close() w, err := os.OpenFile(out, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, info.Mode().Perm()) if err != nil { return err } sum := sha256.New() if _, err := io.Copy(io.MultiWriter(w, sum), in); err != nil { w.Close() return err } if err := w.Close(); err != nil { return err } lines = append(lines, fmt.Sprintf("%s %o %s", rel, info.Mode().Perm()&0o111, hex.EncodeToString(sum.Sum(nil)))) return nil }) if err != nil { return "", err } sort.Strings(lines) sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) return "narrow:" + hex.EncodeToString(sum[:]), nil }