diff --git a/.gitignore b/.gitignore index b6d1dc33..6f38594a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,306 @@ modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image /checks/paths/paths /modules/mesh-build-go/mesh-build-go /modules/mesh-delivery-gate/mesh-delivery-gate + +# A binary built in place, named like its module directory or a cmd directory, in the module root or in that cmd directory +# (issue 1921, try speed S13), for every module; the lint step of every check refuses one a change adds, a module added +# later included. +/modules/adwaita/adwaita +/modules/adwaita/adwaita-tools +/modules/adwaita/cmd/adwaita-tools/adwaita-tools +/modules/artifact-store-tools/artifact-store-tools +/modules/artifact-store-tools/cmd/artifact-store-tools/artifact-store-tools +/modules/asus-zephyrus-g14/asus-zephyrus-g14 +/modules/asus-zephyrus-g14/zephyrus +/modules/asus-zephyrus-g14/cmd/zephyrus/zephyrus +/modules/audit-logger/audit-logger +/modules/avahi/avahi +/modules/avahi/avahi-tools +/modules/avahi/cmd/avahi-tools/avahi-tools +/modules/baserow/baserow +/modules/blueman/blueman +/modules/blueman/blueman-tools +/modules/blueman/cmd/blueman-tools/blueman-tools +/modules/bluetooth/bluetooth +/modules/bluetooth/bluetooth-tools +/modules/bluetooth/cmd/bluetooth-tools/bluetooth-tools +/modules/brother/brother +/modules/brother/cmd/brother/brother +/modules/buildah/buildah +/modules/ca-trust/ca-trust +/modules/claude-code/claude-code +/modules/claude-code/cmd/claude-code/claude-code +/modules/claude-licence-manager/claude-licence-manager +/modules/claude-licence-manager/cmd/claude-licence-manager/claude-licence-manager +/modules/clipmenu/clipmenu +/modules/clipmenu/clipmenu-tools +/modules/clipmenu/cmd/clipmenu-tools/clipmenu-tools +/modules/confluence/confluence +/modules/cups/cups +/modules/cups/cups-tools +/modules/cups/cmd/cups-tools/cups-tools +/modules/dbus/dbus +/modules/dbus/dbus-tools +/modules/dbus/cmd/dbus-tools/dbus-tools +/modules/desk-channel/desk-channel +/modules/desk-channel/cmd/desk-channel/desk-channel +/modules/dev-tools/dev-tools +/modules/disk-load/disk-load +/modules/disk-load/cmd/disk-load/disk-load +/modules/distribution/distribution +/modules/dmenu/dmenu +/modules/dmenu/dmenu-tools +/modules/dmenu/cmd/dmenu-tools/dmenu-tools +/modules/dnsmasq/dnsmasq +/modules/dnsmasq/dnsmasq-tools +/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools +/modules/docker/docker +/modules/docker/docker-tools +/modules/docker/cmd/docker-tools/docker-tools +/modules/docker-compose/docker-compose +/modules/docker-compose/docker-compose-tools +/modules/docker-compose/cmd/docker-compose-tools/docker-compose-tools +/modules/dunst/dunst +/modules/dunst/dunst-tools +/modules/dunst/cmd/dunst-tools/dunst-tools +/modules/fail2ban/fail2ban +/modules/fail2ban/fail2ban-tools +/modules/fail2ban/cmd/fail2ban-tools/fail2ban-tools +/modules/feh/feh +/modules/feh/feh-tools +/modules/feh/cmd/feh-tools/feh-tools +/modules/flatpak/flatpak +/modules/flatpak/flatpak-tools +/modules/flatpak/cmd/flatpak-tools/flatpak-tools +/modules/fonts/fonts +/modules/fonts/fonts-tools +/modules/fonts/cmd/fonts-tools/fonts-tools +/modules/forticlient/forticlient +/modules/forticlient/forticlient-tools +/modules/forticlient/cmd/forticlient-tools/forticlient-tools +/modules/gitea/gitea +/modules/gitea/agent-teams +/modules/gitea/cmd/agent-teams/agent-teams +/modules/gitea/agent-token +/modules/gitea/cmd/agent-token/agent-token +/modules/gitea/forge +/modules/gitea/cmd/forge/forge +/modules/gitea/forge-account-provider +/modules/gitea/cmd/forge-account-provider/forge-account-provider +/modules/gitea/npm-registry +/modules/gitea/cmd/npm-registry/npm-registry +/modules/gitea/repositories +/modules/gitea/cmd/repositories/repositories +/modules/gitlab/gitlab +/modules/gnome-keyring/gnome-keyring +/modules/gnome-keyring/gnome-keyring-tools +/modules/gnome-keyring/cmd/gnome-keyring-tools/gnome-keyring-tools +/modules/go/go +/modules/grafana/grafana +/modules/hello-web/hello-web +/modules/home-assistant/home-assistant +/modules/hostname/hostname +/modules/hostname/hostname-tools +/modules/hostname/cmd/hostname-tools/hostname-tools +/modules/i3/i3 +/modules/i3/i3-tools +/modules/i3/cmd/i3-tools/i3-tools +/modules/i3status-rust/i3status-rust +/modules/i3status-rust/i3status-rust-tools +/modules/i3status-rust/cmd/i3status-rust-tools/i3status-rust-tools +/modules/icecast/icecast +/modules/influxdb/influxdb +/modules/invoicing/invoicing +/modules/jira/jira +/modules/kernel/kernel +/modules/kernel/kernel-tools +/modules/kernel/cmd/kernel-tools/kernel-tools +/modules/keycloak/keycloak +/modules/keycloak/keycloak-provider +/modules/keycloak/cmd/keycloak-provider/keycloak-provider +/modules/lab/lab +/modules/lemurs/lemurs +/modules/lemurs/lemurs-tools +/modules/lemurs/cmd/lemurs-tools/lemurs-tools +/modules/letta/letta +/modules/localization/localization +/modules/localization/localization-tools +/modules/localization/cmd/localization-tools/localization-tools +/modules/local-model-consumer/local-model-consumer +/modules/logrotate/logrotate +/modules/logrotate/logrotate-tools +/modules/logrotate/cmd/logrotate-tools/logrotate-tools +/modules/mailu/mailu +/modules/mailu/automx +/modules/mailu/cmd/automx/automx +/modules/marrytts/marrytts +/modules/matrix/matrix +/modules/memory-pressure/memory-pressure +/modules/memory-pressure/cmd/memory-pressure/memory-pressure +/modules/mesh-build-go/mesh-build-go +/modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go +/modules/mesh-build-image/mesh-build-image +/modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image +/modules/mesh-build-node/mesh-build-node +/modules/mesh-build-node/cmd/mesh-build-node/mesh-build-node +/modules/mesh-catalog/mesh-catalog +/modules/mesh-cli/mesh-cli +/modules/mesh-cli/cmd/mesh-cli/mesh-cli +/modules/mesh-config/mesh-config +/modules/mesh-config/cmd/mesh-config/mesh-config +/modules/mesh-delivery-gate/mesh-delivery-gate +/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/mesh-delivery-gate +/modules/mesh-files/mesh-files +/modules/mesh-files/cmd/mesh-files/mesh-files +/modules/mesh-issues/mesh-issues +/modules/mesh-issues/cmd/mesh-issues/mesh-issues +/modules/mesh-knowledge/mesh-knowledge +/modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge +/modules/mesh-mail/mesh-mail +/modules/mesh-mail/cmd/mesh-mail/mesh-mail +/modules/mesh-mail-notifier/mesh-mail-notifier +/modules/mesh-mail-notifier/cmd/mesh-mail-notifier/mesh-mail-notifier +/modules/mesh-mail-reader/mesh-mail-reader +/modules/mesh-mail-reader/cmd/mesh-mail-reader/mesh-mail-reader +/modules/mesh-mail-web/mesh-mail-web +/modules/mesh-mail-web/cmd/mesh-mail-web/mesh-mail-web +/modules/mesh-provisioner/mesh-provisioner +/modules/mesh-provisioner/cmd/mesh-provisioner/mesh-provisioner +/modules/mesh-provisioner-client/mesh-provisioner-client +/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client +/modules/mesh-reads/mesh-reads +/modules/mesh-reads/cmd/mesh-reads/mesh-reads +/modules/mesh-vault/mesh-vault +/modules/mesh-vault/cmd/mesh-vault/mesh-vault +/modules/mesh-watcher/mesh-watcher +/modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher +/modules/messenger/messenger +/modules/messenger/cmd/messenger/messenger +/modules/minio/minio +/modules/minio/object-store +/modules/minio/cmd/object-store/object-store +/modules/model-usage/model-usage +/modules/mongodb/mongodb +/modules/mosquitto/mosquitto +/modules/mounts/mounts +/modules/mounts/cmd/mounts/mounts +/modules/mssql/mssql +/modules/n8n/n8n +/modules/nats/nats +/modules/nats/mesh-nats-snapshot +/modules/nats/cmd/mesh-nats-snapshot/mesh-nats-snapshot +/modules/nats/nats-tools +/modules/nats/cmd/nats-tools/nats-tools +/modules/netcheck/netcheck +/modules/netcheck/cmd/netcheck/netcheck +/modules/networkmanager/networkmanager +/modules/networkmanager/uplink-tools +/modules/networkmanager/cmd/uplink-tools/uplink-tools +/modules/nextcloud/nextcloud +/modules/nextcloud-client/nextcloud-client +/modules/nextcloud-client/nextcloud-client-tools +/modules/nextcloud-client/cmd/nextcloud-client-tools/nextcloud-client-tools +/modules/nfs-server/nfs-server +/modules/nfs-server/cmd/nfs-server/nfs-server +/modules/nftables/nftables +/modules/nm-applet/nm-applet +/modules/nm-applet/nm-applet-tools +/modules/nm-applet/cmd/nm-applet-tools/nm-applet-tools +/modules/node-env/node-env +/modules/nodejs/nodejs +/modules/nodered/nodered +/modules/ollama/ollama +/modules/only-office/only-office +/modules/only-office/cmd/only-office/only-office +/modules/openai-consumer/openai-consumer +/modules/openrazer/openrazer +/modules/openrazer/openrazer-tools +/modules/openrazer/cmd/openrazer-tools/openrazer-tools +/modules/pacman/pacman +/modules/pacman/pacman-tools +/modules/pacman/cmd/pacman-tools/pacman-tools +/modules/photos-eef/photos-eef +/modules/photos-filip/photos-filip +/modules/picom/picom +/modules/picom/picom-tools +/modules/picom/cmd/picom-tools/picom-tools +/modules/polychromatic/polychromatic +/modules/polychromatic/polychromatic-tools +/modules/polychromatic/cmd/polychromatic-tools/polychromatic-tools +/modules/postgres/postgres +/modules/postgres/postgres-provider +/modules/postgres/cmd/postgres-provider/postgres-provider +/modules/power/power +/modules/power/cmd/power/power +/modules/powerlevel10k/powerlevel10k +/modules/records/records +/modules/records/cmd/records/records +/modules/redis/redis +/modules/restic/restic +/modules/restic/restic-backups +/modules/restic/cmd/restic-backups/restic-backups +/modules/rofi/rofi +/modules/rofi/rofi-tools +/modules/rofi/cmd/rofi-tools/rofi-tools +/modules/route-adapter/route-adapter +/modules/route-adapter/route-provider +/modules/route-adapter/cmd/route-provider/route-provider +/modules/screen-lock/screen-lock +/modules/screen-lock/screen-lock-tools +/modules/screen-lock/cmd/screen-lock-tools/screen-lock-tools +/modules/searxng/searxng +/modules/sensors/sensors +/modules/sensors/cmd/sensors/sensors +/modules/showcase/showcase +/modules/slack/slack +/modules/slack/slack-tools +/modules/slack/cmd/slack-tools/slack-tools +/modules/snapd/snapd +/modules/snapd/snapd-tools +/modules/snapd/cmd/snapd-tools/snapd-tools +/modules/ssh-client/ssh-client +/modules/ssh-client/ssh-client-tools +/modules/ssh-client/cmd/ssh-client-tools/ssh-client-tools +/modules/sshd/sshd +/modules/step-ca/step-ca +/modules/sudo/sudo +/modules/sudo/sudo-tools +/modules/sudo/cmd/sudo-tools/sudo-tools +/modules/supabase/supabase +/modules/systemd/systemd +/modules/systemd/systemd-tools +/modules/systemd/cmd/systemd-tools/systemd-tools +/modules/systemd-networkd/systemd-networkd +/modules/systemd-networkd/uplink-tools +/modules/systemd-networkd/cmd/uplink-tools/uplink-tools +/modules/systemd-resolved/systemd-resolved +/modules/systemd-resolved/resolver-tools +/modules/systemd-resolved/cmd/resolver-tools/resolver-tools +/modules/telegram/telegram +/modules/telegram/cmd/telegram/telegram +/modules/time-sync/time-sync +/modules/time-sync/time-sync-tools +/modules/time-sync/cmd/time-sync-tools/time-sync-tools +/modules/triggerhappy/triggerhappy +/modules/triggerhappy/triggerhappy-tools +/modules/triggerhappy/cmd/triggerhappy-tools/triggerhappy-tools +/modules/umami/umami +/modules/unifi/unifi +/modules/website/website +/modules/xclip/xclip +/modules/xclip/xclip-tools +/modules/xclip/cmd/xclip-tools/xclip-tools +/modules/xdg/xdg +/modules/xdg/xdg-tools +/modules/xdg/cmd/xdg-tools/xdg-tools +/modules/xorg/xorg +/modules/xorg/cmd/xorg/xorg +/modules/xterm/xterm +/modules/xterm/xterm-tools +/modules/xterm/cmd/xterm-tools/xterm-tools +/modules/zfs/zfs +/modules/zfs/zfs-tools +/modules/zfs/cmd/zfs-tools/zfs-tools +/modules/zsh/zsh +/modules/zsh-autosuggestions/zsh-autosuggestions +/modules/zsh-syntax-highlighting/zsh-syntax-highlighting diff --git a/modules/forticlient/cmd/forticlient-tools/forticlient-tools b/modules/forticlient/cmd/forticlient-tools/forticlient-tools deleted file mode 100755 index 4588002a..00000000 Binary files a/modules/forticlient/cmd/forticlient-tools/forticlient-tools and /dev/null differ diff --git a/modules/mesh-build-go/cmd/mesh-build-go/binaries.go b/modules/mesh-build-go/cmd/mesh-build-go/binaries.go new file mode 100644 index 00000000..bfb7fb08 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/binaries.go @@ -0,0 +1,90 @@ +package main + +// Stray build output (issue 1921, try speed S13): the lint step fails a change that adds an executable (ELF or Mach-O) or +// a file over MaxAddedFile, naming each, unless the repository's .binaries-allowed lists its path. Build binaries were +// committed three times in one night; the forge keeps them for ever. + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// MaxAddedFile is the largest file a change may add without .binaries-allowed naming it. +const MaxAddedFile = 5 << 20 + +// BinariesAllowed is the file in a repository's root listing, one path per line ("#" starts a comment), the executables +// and large files it keeps on purpose. +const BinariesAllowed = ".binaries-allowed" + +// executableMagic are the first bytes of an ELF and of each Mach-O form (32 and 64 bits, both byte orders, universal). +var executableMagic = [][]byte{[]byte("\x7fELF"), {0xfe, 0xed, 0xfa, 0xce}, {0xfe, 0xed, 0xfa, 0xcf}, {0xce, 0xfa, 0xed, 0xfe}, + {0xcf, 0xfa, 0xed, 0xfe}, {0xca, 0xfe, 0xba, 0xbe}} + +// addedBinaries is the lint finding of the files the merge adds to the trunk that are executables or over MaxAddedFile, +// nil when there is none. The paths come from git (the merge commit against its first parent, the trunk); a tree with no +// merge (no first parent) adds nothing to judge. +func (m *Module) addedBinaries(ctx context.Context, run forgekey.Runner, tree string) []Failure { + r := run(ctx, []string{"GIT_TERMINAL_PROMPT=0"}, "git", "-C", tree, "diff", "--name-only", "--no-renames", "--diff-filter=A", "HEAD^1", "HEAD") + if r.Status != 0 || r.Err != "" { + return nil + } + allowed := map[string]bool{} + if raw, err := os.ReadFile(filepath.Join(tree, BinariesAllowed)); err == nil { + scan := bufio.NewScanner(bytes.NewReader(raw)) + for scan.Scan() { + if l := strings.TrimSpace(scan.Text()); l != "" && !strings.HasPrefix(l, "#") { + allowed[filepath.ToSlash(l)] = true + } + } + } + var found []string + for _, p := range strings.Split(r.Stdout, "\n") { + if p = strings.TrimSpace(p); p == "" || allowed[p] { + continue + } + path := filepath.Join(tree, filepath.FromSlash(p)) + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() { + continue // narrowed away, or not a file + } + if info.Size() > MaxAddedFile { + found = append(found, fmt.Sprintf("%s (%d bytes, over %d)", p, info.Size(), MaxAddedFile)) + continue + } + if isExecutable(path) { + found = append(found, p+" (an executable)") + } + } + if len(found) == 0 { + return nil + } + sort.Strings(found) + return []Failure{{Package: ".", Test: "binaries", Step: StepLint, + Output: "the change adds build output or a large file (remove it, or list its path in " + BinariesAllowed + "): " + strings.Join(found, ", ")}} +} + +func isExecutable(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + head := make([]byte, 4) + if n, _ := f.Read(head); n < 4 { + return false + } + for _, magic := range executableMagic { + if bytes.Equal(head, magic) { + return true + } + } + return false +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/binaries_test.go b/modules/mesh-build-go/cmd/mesh-build-go/binaries_test.go new file mode 100644 index 00000000..40456a1a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/binaries_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Issue 1921, try speed S13: the lint step fails a change that adds an executable (ELF or Mach-O) or a file over 5 MB, +// naming it, unless .binaries-allowed in the repository's root lists its path; a file the change does not add is not +// judged, and neither is an allowed one. +func TestTheLintStepFailsAChangeThatAddsABinaryOrALargeFile(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "slow") + tree := filepath.Join(m.Work, ws, "tree") + write := func(name string, body []byte) { + os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755) + if err := os.WriteFile(filepath.Join(tree, name), body, 0o644); err != nil { + t.Fatal(err) + } + } + write("go.mod", []byte("module fixture/slow\n")) + elf := append([]byte("\x7fELF"), make([]byte, 64)...) + macho := append([]byte{0xcf, 0xfa, 0xed, 0xfe}, make([]byte, 64)...) + write("modules/x/cmd/x/x", elf) + write("modules/y/y", macho) + write("big.tar", make([]byte, 5<<20+1)) + write("tools/allowed-tool", elf) + write("old/binary", elf) // there before the change + write("modules/x/doc.md", []byte("text\n")) + write(".binaries-allowed", []byte("# kept on purpose\ntools/allowed-tool\n")) + inner := m.Run + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + if name == "git" && strings.Contains(strings.Join(args, " "), "--diff-filter=A") { + return forgekey.Ran{Stdout: "modules/x/cmd/x/x\nmodules/y/y\nbig.tar\ntools/allowed-tool\nmodules/x/doc.md\n.binaries-allowed\n"} + } + return inner(ctx, env, name, args...) + } + res, st := runUnit(t, m, states, ws) + if st.State != delivery.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + var found []Failure + for _, f := range res.Failed { + if f.Test == "binaries" { + found = append(found, f) + } + } + if res.Passed || len(found) != 1 || found[0].Step != StepLint { + t.Fatalf("failed = %+v, want one lint finding of the binaries", res.Failed) + } + out := found[0].Output + for _, want := range []string{"modules/x/cmd/x/x", "modules/y/y", "big.tar"} { + if !strings.Contains(out, want) { + t.Errorf("the finding does not name %s: %s", want, out) + } + } + for _, not := range []string{"allowed-tool", "old/binary", "doc.md"} { + if strings.Contains(out, not) { + t.Errorf("the finding names %s: %s", not, out) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index d2354b59..f8e1256b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -240,6 +240,7 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR lintFailed, lintErr = m.lintAll(ctx, compile, tree, todo, func(d string) []string { return append(m.goEnvFor(filepath.Join(tree, d)), o.env...) }, byDir, slots.acquire) + lintFailed = append(m.addedBinaries(ctx, m.Run, tree), lintFailed...) // git alone, no code of the change res.Lint = m.clock().Sub(linted).Seconds() }() } diff --git a/modules/networkmanager/uplink-tools b/modules/networkmanager/uplink-tools deleted file mode 100755 index cc197674..00000000 Binary files a/modules/networkmanager/uplink-tools and /dev/null differ