Merge pull request 'The catalogue's check refuses a compiled program under modules/, and main tracks none (issue 1577)' (#507) from fix/1577-no-committed-binaries into main

.gitignore

@@ -35,3 +35,7 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
 /checks/paths/paths
 /modules/mesh-build-go/mesh-build-go
 /modules/mesh-delivery-gate/mesh-delivery-gate
+
+# Built by their modules' build step (issue 1577), never committed.
+modules/forticlient/cmd/forticlient-tools/forticlient-tools
+modules/networkmanager/uplink-tools

checks/nobinaries/go.mod

@@ -0,0 +1,3 @@
+module git.novox.be/novox/mesh-catalog/checks/nobinaries
+
+go 1.22

checks/nobinaries/nobinaries.go

@@ -0,0 +1,78 @@
+// Package nobinaries refuses a compiled program under modules/ (issue 1577): it is build output, made by the module's build step,
+// and `go build` in a module's folder writes one beside go.mod where `git add -A` takes it.
+package nobinaries
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// Kind says what executable format the first bytes of a file are, or "" when they are none.
+func Kind(head []byte) string {
+	switch {
+	case bytes.HasPrefix(head, []byte("\x7fELF")):
+		return "ELF"
+	case bytes.HasPrefix(head, []byte("MZ")):
+		return "PE"
+	case bytes.HasPrefix(head, []byte{0xcf, 0xfa, 0xed, 0xfe}), bytes.HasPrefix(head, []byte{0xce, 0xfa, 0xed, 0xfe}),
+		bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xcf}), bytes.HasPrefix(head, []byte{0xfe, 0xed, 0xfa, 0xce}),
+		bytes.HasPrefix(head, []byte{0xca, 0xfe, 0xba, 0xbe}) && len(head) >= 8 && head[7] < 0x20 && head[6] == 0 && head[5] == 0 && head[4] == 0:
+		return "Mach-O"
+	}
+	return ""
+}
+
+// Files are the files of the catalogue's root that the change holds under modules/: the tracked ones when the root is a git
+// checkout (an ignored binary a local build left is not the change's), else every file there.
+func Files(root string) ([]string, error) {
+	cmd := exec.Command("git", "-C", root, "ls-files", "-z", "--", "modules")
+	if out, err := cmd.Output(); err == nil && len(out) > 0 {
+		var files []string
+		for _, f := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") {
+			if _, err := os.Lstat(filepath.Join(root, f)); err == nil { // a file the change deletes is not there
+				files = append(files, f)
+			}
+		}
+		return files, nil
+	}
+	var files []string
+	err := filepath.WalkDir(filepath.Join(root, "modules"), func(p string, d os.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !d.Type().IsRegular() {
+			return err
+		}
+		rel, _ := filepath.Rel(root, p)
+		files = append(files, rel)
+		return nil
+	})
+	return files, err
+}
+
+// Find names each file of files (paths from root) that is a compiled program, with its format.
+func Find(root string, files []string) ([]string, error) {
+	var found []string
+	for _, f := range files {
+		p := filepath.Join(root, f)
+		if st, err := os.Lstat(p); err != nil || !st.Mode().IsRegular() {
+			continue
+		}
+		fh, err := os.Open(p)
+		if err != nil {
+			return nil, err
+		}
+		head := make([]byte, 8)
+		n, err := io.ReadFull(fh, head)
+		fh.Close()
+		if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+			return nil, err
+		}
+		if k := Kind(head[:n]); k != "" {
+			found = append(found, fmt.Sprintf("%s (%s)", f, k))
+		}
+	}
+	return found, nil
+}

checks/nobinaries/nobinaries_test.go

@@ -0,0 +1,54 @@
+package nobinaries
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestFindNamesEachCompiledProgramAndNothingElse(t *testing.T) {
+	root := t.TempDir()
+	files := map[string][]byte{
+		"modules/a/a":           []byte("\x7fELF\x02\x01\x01\x00rest"),
+		"modules/b/cmd/b/b.exe": []byte("MZ\x90\x00"),
+		"modules/c/c":           {0xcf, 0xfa, 0xed, 0xfe, 7, 0, 0, 1},
+		"modules/d/main.go":     []byte("package main\n"),
+		"modules/e/short":       []byte("\x7f"),
+		"modules/f/empty":       {},
+		"modules/g/Foo.class":   {0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52}, // Java, not Mach-O
+	}
+	var names []string
+	for f, b := range files {
+		p := filepath.Join(root, f)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, b, 0o644); err != nil {
+			t.Fatal(err)
+		}
+		names = append(names, f)
+	}
+	got, err := Find(root, names)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 3 {
+		t.Fatalf("found %v; wanted modules/a/a, modules/b/cmd/b/b.exe and modules/c/c only", got)
+	}
+}
+
+// The catalogue's own tree holds no compiled program: a compiled program is build output, so remove it and add it to .gitignore.
+func TestTheCatalogueHoldsNoCompiledProgram(t *testing.T) {
+	root := filepath.Join("..", "..")
+	files, err := Files(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	found, err := Find(root, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range found {
+		t.Errorf("%s: a compiled program is build output: remove it and add it to .gitignore (issue 1577)", f)
+	}
+}

checks/nobinaries/reads-outside

@@ -0,0 +1 @@
+These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).