279 lines
9.2 KiB
Go
279 lines
9.2 KiB
Go
package main
|
|
|
|
// The digest step (novox/hq ADR 0445 §5, ADR 0444; issue 1291): for each package of a workspace that has tests at a
|
|
// level, one digest of everything those tests read. The checker keys the verdict it keeps by it, so a package whose
|
|
// digest is unchanged is not run again. The tool answers in its call (it is short, ADR 0469 §2).
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// DigestVersion is the form of the digest's text: changed whenever what the digest hashes changes in kind.
|
|
const DigestVersion = "2" // 2: no Go version in the text; the toolchain enters a fingerprint as the agreed version (ADR 0477 §3)
|
|
|
|
// goPackage is the part of `go list -json` a digest reads.
|
|
type goPackage struct {
|
|
ImportPath string
|
|
Dir string
|
|
ForTest string
|
|
Standard bool
|
|
DepOnly bool
|
|
Deps []string
|
|
Module *goModule
|
|
// The files a package is built and tested from.
|
|
GoFiles, CgoFiles, CFiles, CXXFiles, MFiles, HFiles, FFiles, SFiles, SwigFiles, SwigCXXFiles, SysoFiles []string
|
|
TestGoFiles, XTestGoFiles []string
|
|
EmbedFiles, TestEmbedFiles, XTestEmbedFiles []string
|
|
}
|
|
|
|
type goModule struct {
|
|
Path, Version, Sum string
|
|
Main bool
|
|
Replace *goModule
|
|
Dir string
|
|
GoVersion string
|
|
}
|
|
|
|
func (p *goPackage) files() []string {
|
|
var all []string
|
|
for _, l := range [][]string{p.GoFiles, p.CgoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles,
|
|
p.SwigCXXFiles, p.SysoFiles, p.TestGoFiles, p.XTestGoFiles, p.EmbedFiles, p.TestEmbedFiles, p.XTestEmbedFiles} {
|
|
all = append(all, l...)
|
|
}
|
|
sort.Strings(all)
|
|
return slicesCompact(all)
|
|
}
|
|
|
|
func slicesCompact(s []string) []string {
|
|
out := s[:0:0]
|
|
for i, v := range s {
|
|
if i == 0 || v != s[i-1] {
|
|
out = append(out, v)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// hasTests says whether the package has test files at the level go list was asked for.
|
|
func (p *goPackage) hasTests() bool { return len(p.TestGoFiles)+len(p.XTestGoFiles) > 0 }
|
|
|
|
// DigestResult is what a digest step answers: one digest per package that has tests at the level, and the packages
|
|
// that have none.
|
|
type DigestResult struct {
|
|
Level string `json:"level"`
|
|
Packages map[string]string `json:"packages"`
|
|
WithoutTests []string `json:"without_tests,omitempty"`
|
|
}
|
|
|
|
func (m *Module) digestTool() StepTool {
|
|
return StepTool{
|
|
Name: "build_go_digest",
|
|
Description: "One digest per package of a fetched workspace, for one level, of everything that package's tests read: " +
|
|
"the files of each package the tests reach in the module's own tree, the version and checksum of each module " +
|
|
"outside it, the package's test data and embedded files, the Go version, the build tags and, at the " +
|
|
"integration level, the postgres major and nats-server versions. Answers in the call.",
|
|
Input: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"},
|
|
"level": map[string]any{"type": "string", "enum": []string{LevelUnit, LevelIntegration}, "description": "the level"},
|
|
},
|
|
"required": []string{"workspace", "level"},
|
|
},
|
|
Run: m.digest,
|
|
}
|
|
}
|
|
|
|
func (m *Module) digest(ctx context.Context, args map[string]any) (any, error) {
|
|
workspace, _ := args["workspace"].(string)
|
|
level, _ := args["level"].(string)
|
|
if !workspaceOK.MatchString(workspace) {
|
|
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
|
}
|
|
tree := filepath.Join(m.Work, workspace, "tree")
|
|
if _, err := os.Stat(tree); err != nil {
|
|
return nil, fmt.Errorf("no workspace %s: it was never fetched here, or it is older than a day and was removed", workspace)
|
|
}
|
|
var tags []string
|
|
header := []string{"digest " + DigestVersion, "level " + level}
|
|
switch level {
|
|
case LevelUnit:
|
|
case LevelIntegration:
|
|
if _, err := m.checkPostgres(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
tags = []string{"integration"}
|
|
header = append(header, "postgres "+strconv.Itoa(m.Recipe.PostgresMajor), "nats-server "+m.Recipe.NatsServer)
|
|
default:
|
|
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
|
|
}
|
|
header = append(header, "tags "+strings.Join(tags, ","))
|
|
|
|
dirs, err := m.goModDirs(ctx, tree)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res := DigestResult{Level: level, Packages: map[string]string{}}
|
|
for _, d := range dirs {
|
|
pkgs, err := m.listPackages(ctx, tree, d, tags)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := digestModule(&res, pkgs, tree, header); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
sort.Strings(res.WithoutTests)
|
|
return res, nil
|
|
}
|
|
|
|
// listPackages is `go list -test -deps -json` of the directory's packages, by import path.
|
|
func (m *Module) listPackages(ctx context.Context, tree, dir string, tags []string) (map[string]*goPackage, error) {
|
|
abs := filepath.Join(tree, dir)
|
|
args := []string{"-C", abs, "list", "-test", "-deps", "-json"}
|
|
if len(tags) > 0 {
|
|
args = append(args, "-tags", strings.Join(tags, ","))
|
|
}
|
|
args = append(args, "./...")
|
|
r := m.Run(ctx, m.goEnvFor(abs), "go", args...)
|
|
if r.Status != 0 || r.Err != "" {
|
|
return nil, fmt.Errorf("go list in %s: %s", dir, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
|
}
|
|
return parsePackages(r.Stdout)
|
|
}
|
|
|
|
// parsePackages reads `go list -json`'s stream of objects.
|
|
func parsePackages(out string) (map[string]*goPackage, error) {
|
|
pkgs := map[string]*goPackage{}
|
|
dec := json.NewDecoder(strings.NewReader(out))
|
|
for {
|
|
var p goPackage
|
|
if err := dec.Decode(&p); err == io.EOF {
|
|
return pkgs, nil
|
|
} else if err != nil {
|
|
return nil, fmt.Errorf("go list answered something that is not its JSON: %w", err)
|
|
}
|
|
pkgs[p.ImportPath] = &p
|
|
}
|
|
}
|
|
|
|
// digestModule adds to res the digest of each root package of one go.mod directory. A root is a package of the main
|
|
// module that is listed, not only a dependency, and not a variant of another.
|
|
func digestModule(res *DigestResult, pkgs map[string]*goPackage, tree string, header []string) error {
|
|
roots := []string{}
|
|
for path, p := range pkgs {
|
|
if p.ForTest == "" && !p.DepOnly && !p.Standard && !strings.HasSuffix(path, ".test") && p.Module != nil && p.Module.Main {
|
|
roots = append(roots, path)
|
|
}
|
|
}
|
|
sort.Strings(roots)
|
|
for _, path := range roots {
|
|
root := pkgs[path]
|
|
if !root.hasTests() {
|
|
res.WithoutTests = append(res.WithoutTests, path)
|
|
continue
|
|
}
|
|
testMain := pkgs[path+".test"]
|
|
if testMain == nil {
|
|
return fmt.Errorf("go list gave no test binary for %s", path)
|
|
}
|
|
d, err := packageDigest(root, testMain, pkgs, tree, header)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: %w", path, err)
|
|
}
|
|
res.Packages[path] = d
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// packageDigest is the digest of one package: a hash over a sorted text of the header, the files of every package its
|
|
// tests reach in the module's own tree (by their content, under their path in the tree), the version and checksum of
|
|
// every module outside it, and the package's own test data. A package of the standard library is the Go version's.
|
|
func packageDigest(root, testMain *goPackage, pkgs map[string]*goPackage, tree string, header []string) (string, error) {
|
|
reached := map[string]*goPackage{root.ImportPath: root}
|
|
for _, dep := range testMain.Deps {
|
|
if p := pkgs[dep]; p != nil {
|
|
reached[dep] = p
|
|
}
|
|
}
|
|
lines := []string{}
|
|
for _, p := range reached {
|
|
if p.Standard || p.Module == nil {
|
|
continue
|
|
}
|
|
mod := p.Module
|
|
if mod.Replace != nil {
|
|
mod = mod.Replace
|
|
}
|
|
if !mod.Main && mod.Version != "" {
|
|
lines = append(lines, "module "+mod.Path+"@"+mod.Version+" "+mod.Sum)
|
|
continue
|
|
}
|
|
for _, f := range p.files() {
|
|
l, err := fileLine(tree, p, f)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
lines = append(lines, l)
|
|
}
|
|
}
|
|
data := filepath.Join(root.Dir, "testdata")
|
|
if _, err := os.Stat(data); err == nil {
|
|
err := filepath.WalkDir(data, func(path string, e fs.DirEntry, err error) error {
|
|
if err != nil || e.IsDir() {
|
|
return err
|
|
}
|
|
l, err := contentLine(tree, root, path)
|
|
lines = append(lines, l)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
sort.Strings(lines)
|
|
h := sha256.New()
|
|
for _, l := range append(append([]string{}, header...), lines...) {
|
|
h.Write([]byte(l + "\n"))
|
|
}
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
|
|
func fileLine(tree string, p *goPackage, name string) (string, error) {
|
|
return contentLine(tree, p, filepath.Join(p.Dir, name))
|
|
}
|
|
|
|
// contentLine is "file <path> <sha256>": the path is the file's in the tree (the import path and the name when it lies
|
|
// outside, as a local replacement does), so a digest does not depend on where the workspace is.
|
|
func contentLine(tree string, p *goPackage, path string) (string, error) {
|
|
label := p.ImportPath + "/" + filepath.Base(path)
|
|
if rel, err := filepath.Rel(tree, path); err == nil && !strings.HasPrefix(rel, "..") {
|
|
label = filepath.ToSlash(rel)
|
|
}
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
to, _ := os.Readlink(path)
|
|
return "link " + label + " " + to, nil
|
|
}
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sum := sha256.Sum256(raw)
|
|
return "file " + label + " " + hex.EncodeToString(sum[:]), nil
|
|
}
|