mesh-build-go: fail a change that adds an executable or a file over 5 MB in the lint step unless .binaries-allowed lists it; ignore every module's in-place build binary, and remove the two that were committed (issue 1921, try speed S13)

modules/mesh-build-go/cmd/mesh-build-go/binaries.go

@@ -0,0 +1,90 @@
+package main
+
+// Stray build output (issue 1921, try speed S13): the lint step fails a change that adds an executable (ELF or Mach-O) or
+// a file over MaxAddedFile, naming each, unless the repository's .binaries-allowed lists its path. Build binaries were
+// committed three times in one night; the forge keeps them for ever.
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+)
+
+// MaxAddedFile is the largest file a change may add without .binaries-allowed naming it.
+const MaxAddedFile = 5 << 20
+
+// BinariesAllowed is the file in a repository's root listing, one path per line ("#" starts a comment), the executables
+// and large files it keeps on purpose.
+const BinariesAllowed = ".binaries-allowed"
+
+// executableMagic are the first bytes of an ELF and of each Mach-O form (32 and 64 bits, both byte orders, universal).
+var executableMagic = [][]byte{[]byte("\x7fELF"), {0xfe, 0xed, 0xfa, 0xce}, {0xfe, 0xed, 0xfa, 0xcf}, {0xce, 0xfa, 0xed, 0xfe},
+	{0xcf, 0xfa, 0xed, 0xfe}, {0xca, 0xfe, 0xba, 0xbe}}
+
+// addedBinaries is the lint finding of the files the merge adds to the trunk that are executables or over MaxAddedFile,
+// nil when there is none. The paths come from git (the merge commit against its first parent, the trunk); a tree with no
+// merge (no first parent) adds nothing to judge.
+func (m *Module) addedBinaries(ctx context.Context, run forgekey.Runner, tree string) []Failure {
+	r := run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", tree, "diff", "--name-only", "--no-renames", "--diff-filter=A", "HEAD^1", "HEAD")
+	if r.Status != 0 || r.Err != "" {
+		return nil
+	}
+	allowed := map[string]bool{}
+	if raw, err := os.ReadFile(filepath.Join(tree, BinariesAllowed)); err == nil {
+		scan := bufio.NewScanner(bytes.NewReader(raw))
+		for scan.Scan() {
+			if l := strings.TrimSpace(scan.Text()); l != "" && !strings.HasPrefix(l, "#") {
+				allowed[filepath.ToSlash(l)] = true
+			}
+		}
+	}
+	var found []string
+	for _, p := range strings.Split(r.Stdout, "\n") {
+		if p = strings.TrimSpace(p); p == "" || allowed[p] {
+			continue
+		}
+		path := filepath.Join(tree, filepath.FromSlash(p))
+		info, err := os.Lstat(path)
+		if err != nil || !info.Mode().IsRegular() {
+			continue // narrowed away, or not a file
+		}
+		if info.Size() > MaxAddedFile {
+			found = append(found, fmt.Sprintf("%s (%d bytes, over %d)", p, info.Size(), MaxAddedFile))
+			continue
+		}
+		if isExecutable(path) {
+			found = append(found, p+" (an executable)")
+		}
+	}
+	if len(found) == 0 {
+		return nil
+	}
+	sort.Strings(found)
+	return []Failure{{Package: ".", Test: "binaries", Step: StepLint,
+		Output: "the change adds build output or a large file (remove it, or list its path in " + BinariesAllowed + "): " + strings.Join(found, ", ")}}
+}
+
+func isExecutable(path string) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+	head := make([]byte, 4)
+	if n, _ := f.Read(head); n < 4 {
+		return false
+	}
+	for _, magic := range executableMagic {
+		if bytes.Equal(head, magic) {
+			return true
+		}
+	}
+	return false
+}

modules/mesh-build-go/cmd/mesh-build-go/binaries_test.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+)
+
+// Issue 1921, try speed S13: the lint step fails a change that adds an executable (ELF or Mach-O) or a file over 5 MB,
+// naming it, unless .binaries-allowed in the repository's root lists its path; a file the change does not add is not
+// judged, and neither is an allowed one.
+func TestTheLintStepFailsAChangeThatAddsABinaryOrALargeFile(t *testing.T) {
+	m, states, ws, _ := workspaceFor(t, "slow")
+	tree := filepath.Join(m.Work, ws, "tree")
+	write := func(name string, body []byte) {
+		os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755)
+		if err := os.WriteFile(filepath.Join(tree, name), body, 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("go.mod", []byte("module fixture/slow\n"))
+	elf := append([]byte("\x7fELF"), make([]byte, 64)...)
+	macho := append([]byte{0xcf, 0xfa, 0xed, 0xfe}, make([]byte, 64)...)
+	write("modules/x/cmd/x/x", elf)
+	write("modules/y/y", macho)
+	write("big.tar", make([]byte, 5<<20+1))
+	write("tools/allowed-tool", elf)
+	write("old/binary", elf) // there before the change
+	write("modules/x/doc.md", []byte("text\n"))
+	write(".binaries-allowed", []byte("# kept on purpose\ntools/allowed-tool\n"))
+	inner := m.Run
+	m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		if name == "git" && strings.Contains(strings.Join(args, " "), "--diff-filter=A") {
+			return forgekey.Ran{Stdout: "modules/x/cmd/x/x\nmodules/y/y\nbig.tar\ntools/allowed-tool\nmodules/x/doc.md\n.binaries-allowed\n"}
+		}
+		return inner(ctx, env, name, args...)
+	}
+	res, st := runUnit(t, m, states, ws)
+	if st.State != delivery.StepPassed {
+		t.Fatalf("the step ended %s: %s", st.State, st.Reason)
+	}
+	var found []Failure
+	for _, f := range res.Failed {
+		if f.Test == "binaries" {
+			found = append(found, f)
+		}
+	}
+	if res.Passed || len(found) != 1 || found[0].Step != StepLint {
+		t.Fatalf("failed = %+v, want one lint finding of the binaries", res.Failed)
+	}
+	out := found[0].Output
+	for _, want := range []string{"modules/x/cmd/x/x", "modules/y/y", "big.tar"} {
+		if !strings.Contains(out, want) {
+			t.Errorf("the finding does not name %s: %s", want, out)
+		}
+	}
+	for _, not := range []string{"allowed-tool", "old/binary", "doc.md"} {
+		if strings.Contains(out, not) {
+			t.Errorf("the finding names %s: %s", not, out)
+		}
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/test.go

@@ -240,6 +240,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
 			lintFailed, lintErr = m.lintAll(ctx, compile, tree, todo, func(d string) []string {
 				return append(m.goEnvFor(filepath.Join(tree, d)), o.env...)
 			}, byDir, slots.acquire)
+			lintFailed = append(m.addedBinaries(ctx, m.Run, tree), lintFailed...) // git alone, no code of the change
 			res.Lint = m.clock().Sub(linted).Seconds()
 		}()
 	}