Files
mesh-catalog/modules/docker/cmd/docker-tools/docker_test.go
T
jochen 3d8b52c9f4 docker: say where the runtime's event history begins, so a short history never reads as a quiet one (hq issue 348)
docker_events asked for three hours across the 2026-10-09 resolver outage on the control node answered
zero events: the runtime keeps only its last ~256 events in memory, and health-check execs filled them
with the last ten minutes. The answer now carries held, oldest_held and complete, and a history line
when the window reaches further back than what the runtime still holds. The type filter moves into the
tool, since the history's start is only known from all of it.
2026-10-09 15:04:45 +02:00

416 lines
18 KiB
Go

package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"reflect"
"strings"
"sync"
"testing"
"time"
)
type call struct {
name string
args []string
}
// fake answers each command by the first rule whose prefix matches "name arg arg…".
type fake struct {
mu sync.Mutex
rules []rule
calls []call
}
type rule struct {
prefix string
ran Ran
}
func (f *fake) on(prefix string, r Ran) *fake { f.rules = append(f.rules, rule{prefix, r}); return f }
func (f *fake) run(_ context.Context, name string, args ...string) Ran {
f.mu.Lock()
defer f.mu.Unlock()
f.calls = append(f.calls, call{name, args})
line := strings.Join(append([]string{name}, args...), " ")
for _, r := range f.rules {
if strings.HasPrefix(line, r.prefix) {
return r.ran
}
}
return Ran{Status: 1, Stderr: "unexpected: " + line}
}
func (f *fake) ran(prefix string) bool {
for _, c := range f.calls {
if strings.HasPrefix(strings.Join(append([]string{c.name}, c.args...), " "), prefix) {
return true
}
}
return false
}
func client(f *fake, uid int) *Client {
return &Client{Run: f.run, UID: uid, ReadFile: func(string) ([]byte, error) { return nil, fs.ErrNotExist },
Now: func() time.Time { return time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) }}
}
const held = `{"Id":"aaaaaaaaaaaaaaaa","Name":"/mesh-web","Created":"2026-10-01T00:00:00Z","Image":"sha256:img1",
"Config":{"Image":"web:1","Labels":{"mesh-host.id":"hello-web.server","mesh-host.spec":"x"},"Env":["PASSWORD=hunter2","PATH=/bin"]},
"State":{"Status":"running","Running":true,"StartedAt":"2026-10-01T00:00:01Z","FinishedAt":"0001-01-01T00:00:00Z","Health":{"Status":"healthy"}},
"HostConfig":{"RestartPolicy":{"Name":"unless-stopped"},"NetworkMode":"bridge"},
"NetworkSettings":{"Ports":{"80/tcp":[{"HostIp":"0.0.0.0","HostPort":"8080"}]}},
"Mounts":[{"Type":"volume","Name":"webdata","Destination":"/data","RW":true}]}`
const stray = `{"Id":"bbbbbbbbbbbbbbbb","Name":"/dev-db","Created":"2026-09-01T00:00:00Z","Image":"sha256:img2",
"Config":{"Image":"postgres:16","Labels":{"com.docker.compose.project":"dev","com.docker.compose.project.working_dir":"/home/op/dev"}},
"State":{"Status":"exited","ExitCode":1,"FinishedAt":"2026-09-02T00:00:00Z"},
"HostConfig":{"RestartPolicy":{"Name":"no"}},"NetworkSettings":{"Ports":{}},
"Mounts":[{"Type":"volume","Name":"dbdata","Destination":"/var/lib/postgresql/data","RW":true}]}`
func machine() *fake {
return (&fake{}).
on("docker ps --all --quiet --no-trunc", Ran{Stdout: "aaaaaaaaaaaaaaaa\nbbbbbbbbbbbbbbbb\n"}).
on("docker container inspect aaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbb", Ran{Stdout: "[" + held + "," + stray + "]"}).
on("docker container inspect mesh-web", Ran{Stdout: "[" + held + "]"}).
on("docker container inspect dev-db", Ran{Stdout: "[" + stray + "]"})
}
func TestARefusedSocketIsAskedAgainThroughSudoWithoutAPromptUnlessThisIsRoot(t *testing.T) {
denied := Ran{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get ...: dial unix /var/run/docker.sock: connect: permission denied\n"}
f := (&fake{}).on("docker ", denied).on("sudo -n docker info", Ran{Stdout: "{}"})
if _, err := client(f, 1000).docker(context.Background(), "info", "--format", "{{json .}}"); err != nil {
t.Fatal(err)
}
if !f.ran("sudo -n docker info --format") {
t.Fatalf("not escalated: %+v", f.calls)
}
f = (&fake{}).on("docker ", denied)
if _, err := client(f, 0).docker(context.Background(), "info"); err == nil || f.ran("sudo") {
t.Fatalf("root escalated or answered: %v %+v", err, f.calls)
}
}
func TestFailuresAreNamedByHowTheyFailed(t *testing.T) {
denied := Ran{Status: 1, Stderr: "permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock\n"}
cases := map[string]*fake{
"may not escalate without a prompt": (&fake{}).on("docker ", denied).on("sudo ", Ran{Status: 1, Stderr: "sudo: a password is required\n"}),
"sudo is not installed": (&fake{}).on("docker ", denied).on("sudo ", Ran{Status: 127, Err: "ENOENT"}),
"docker is not installed": (&fake{}).on("docker ", Ran{Status: 127, Err: "ENOENT"}),
"daemon is not answering": (&fake{}).on("docker ", Ran{Status: 1, Stderr: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\n"}),
"did not answer: no answer within": (&fake{}).on("docker ", Ran{Status: 124, Err: "no answer within 20 s"}),
"docker info failed (3): boom": (&fake{}).on("docker ", Ran{Status: 3, Stderr: "boom\n"}),
}
for want, f := range cases {
_, err := client(f, 1000).docker(context.Background(), "info")
if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("want %q, got %v", want, err)
}
}
}
func TestANameIsNeverAnOption(t *testing.T) {
for _, bad := range []string{"--help", "-v", "", "a b", "x;y"} {
if _, err := Ref(bad); err == nil {
t.Errorf("%q accepted", bad)
}
}
for _, good := range []string{"mesh-web", "aaaaaaaaaaaa", "registry.mesh.internal:5100/x@sha256:abc", "dev_db.1"} {
if _, err := Ref(good); err != nil {
t.Errorf("%q refused: %v", good, err)
}
}
f := machine()
for _, verb := range []string{"start", "stop", "restart"} {
if _, err := client(f, 1000).Act(context.Background(), verb, "--rm"); err == nil {
t.Errorf("%s took an option", verb)
}
}
if len(f.calls) != 0 {
t.Fatalf("docker was called: %+v", f.calls)
}
}
func TestEveryContainerIsListedAndTheMeshsAreMarked(t *testing.T) {
c := client(machine(), 1000)
all, err := c.Containers(context.Background(), "", "", "")
if err != nil || len(all) != 2 {
t.Fatalf("%v %+v", err, all)
}
web, db := all[1], all[0]
if !web.MeshHeld || web.HeldBy != "hello-web.server" || web.Module != "hello-web" || web.Health != "healthy" {
t.Errorf("held: %+v", web)
}
if !reflect.DeepEqual(web.Ports, []string{"0.0.0.0:8080->80/tcp"}) || web.Mounts[0].Name != "webdata" {
t.Errorf("ports/mounts: %+v", web)
}
if db.MeshHeld || db.Compose != "dev" || db.ComposeDir != "/home/op/dev" || db.FinishedAt == "" {
t.Errorf("stray: %+v", db)
}
mesh, _ := c.Containers(context.Background(), "mesh", "", "")
other, _ := c.Containers(context.Background(), "other", "", "")
if len(mesh) != 1 || mesh[0].Name != "mesh-web" || len(other) != 1 || other[0].Name != "dev-db" {
t.Errorf("held filter: %+v / %+v", mesh, other)
}
if _, err := c.Containers(context.Background(), "mine", "", ""); err == nil {
t.Error("an unknown held filter was accepted")
}
}
func TestNoContainersIsAnEmptyListAndAFailureIsAnError(t *testing.T) {
got, err := client((&fake{}).on("docker ps", Ran{}), 1000).Containers(context.Background(), "", "", "")
if err != nil || got == nil || len(got) != 0 {
t.Fatalf("%v %v", got, err)
}
if _, err := client((&fake{}).on("docker ps", Ran{Status: 1, Stderr: "Cannot connect to the Docker daemon\n"}), 1000).Containers(context.Background(), "", "", ""); err == nil {
t.Fatal("a daemon that does not answer read as no containers")
}
}
func TestInspectLeavesTheEnvironmentsValuesOut(t *testing.T) {
got, err := client(machine(), 1000).Inspect(context.Background(), "mesh-web")
if err != nil {
t.Fatal(err)
}
b, _ := json.Marshal(got)
if strings.Contains(string(b), "hunter2") || !strings.Contains(string(b), `"PASSWORD"`) || got["mesh_held"] != true {
t.Fatalf("%s", b)
}
}
func TestActingOnAMeshContainerSaysTheHostRestoresIt(t *testing.T) {
f := machine().on("docker stop", Ran{}).on("docker start", Ran{})
got, err := client(f, 1000).Act(context.Background(), "stop", "mesh-web")
if err != nil {
t.Fatal(err)
}
if !f.ran("docker stop --time 10 mesh-web") || got["mesh_held"] != true || !strings.Contains(got["note"].(string), "node-engine restores") {
t.Fatalf("%v %+v", got, f.calls)
}
got, _ = client(f, 1000).Act(context.Background(), "start", "dev-db")
if _, noted := got["note"]; noted || got["mesh_held"] != false {
t.Fatalf("a stray was noted: %v", got)
}
}
func TestPruneIsADryRunByDefaultAndNeverTouchesAVolumeOrAMeshContainer(t *testing.T) {
f := machine().
on("docker image ls --no-trunc --filter dangling=true", Ran{Stdout: `{"ID":"sha256:dead","Size":"1.5GB"}` + "\n"}).
on("docker system df --format", Ran{Stdout: `{"Type":"Build Cache","TotalCount":"3","Size":"2GB","Reclaimable":"1GB"}` + "\n"}).
on("docker image prune", Ran{Stdout: "Deleted Images:\nx\n\nTotal reclaimed space: 1.5GB\n"}).
on("docker builder prune", Ran{Stdout: "Total:\t1GB\n"}).
on("docker container rm", Ran{})
c := client(f, 1000)
got, err := c.Prune(context.Background(), PruneAsk{Images: true, BuildCache: true, Containers: true, DryRun: true})
if err != nil {
t.Fatal(err)
}
if f.ran("docker image prune") || f.ran("docker builder prune") || f.ran("docker container rm") {
t.Fatalf("a dry run removed something: %+v", f.calls)
}
if got["images"].(map[string]any)["dangling"] != 1 || !reflect.DeepEqual(got["containers"].(map[string]any)["stopped_not_held"], []string{"dev-db"}) {
t.Fatalf("%v", got)
}
got, err = c.Prune(context.Background(), PruneAsk{Images: true, BuildCache: true, Containers: true, OlderThanH: 24})
if err != nil {
t.Fatal(err)
}
if !f.ran("docker image prune --force --filter until=24h") || !f.ran("docker builder prune --force --filter until=24h") || !f.ran("docker container rm dev-db") {
t.Fatalf("not pruned: %+v", f.calls)
}
for _, c := range f.calls {
line := strings.Join(c.args, " ")
if strings.Contains(line, "volume") || strings.Contains(line, "mesh-web") && c.args[0] != "container" || strings.Contains(line, "--volumes") || strings.Contains(line, "--all") && c.args[0] != "ps" {
t.Errorf("prune reached too far: %s", line)
}
}
if got["images"].(map[string]any)["reclaimed"] != "1.5GB" || got["build_cache"].(map[string]any)["reclaimed"] != "1GB" {
t.Errorf("reclaimed: %v", got)
}
}
func TestLogsMergeBothStreamsInOrderAndKeepTheTail(t *testing.T) {
f := (&fake{}).on("docker logs", Ran{Stdout: "2026-10-04T10:00:01Z out one\n2026-10-04T10:00:03Z out two\n", Stderr: "2026-10-04T10:00:02Z err one\n"})
got, err := client(f, 1000).Logs(context.Background(), "web", 2, "30m")
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(got["lines"], []string{"2026-10-04T10:00:02Z err one", "2026-10-04T10:00:03Z out two"}) {
t.Fatalf("%v", got["lines"])
}
if !f.ran("docker logs --timestamps --tail 2 --since 30m web") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Logs(context.Background(), "web", 2, "--follow"); err == nil {
t.Fatal("since took an option")
}
f = (&fake{}).on("docker logs", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"})
if _, err := client(f, 1000).Logs(context.Background(), "nope", 2, ""); err == nil {
t.Fatal("a missing container read as no lines")
}
}
func TestSizesAreReadAsDockerPrintsThem(t *testing.T) {
for in, want := range map[string]int64{"0B": 0, "55.63GB": 55630000000, "33.2MiB": 34812723, "1.5kB": 1500, "12MB (34%)": 12000000, "N/A": -1} {
if got := Bytes(in); got != want {
t.Errorf("%s: %d, want %d", in, got, want)
}
}
}
func TestDaemonConfigSaysWhatTheDaemonHasNotTakenYet(t *testing.T) {
f := (&fake{}).on("docker info", Ran{Stdout: `{"ServerVersion":"29.8.2","LiveRestoreEnabled":false,"LoggingDriver":"json-file","RegistryConfig":{"IndexConfigs":{"docker.io":{"Secure":true},"registry.mesh.internal:5100":{"Secure":false}}}}`})
c := client(f, 1000)
c.ReadFile = func(string) ([]byte, error) {
return []byte(`{"live-restore": true, "dns": ["10.0.0.1"], "log-driver": "local"}`), nil
}
got, err := c.DaemonConfig(context.Background())
if err != nil {
t.Fatal(err)
}
pending := strings.Join(got["pending"].([]string), "\n")
if !strings.Contains(pending, "live-restore is true in the file and false") || !strings.Contains(pending, "log-driver is local") {
t.Errorf("pending: %s", pending)
}
if !reflect.DeepEqual(got["daemon"].(map[string]any)["InsecureRegistries"], []string{"registry.mesh.internal:5100"}) {
t.Errorf("registries: %v", got["daemon"])
}
if !reflect.DeepEqual(got["read_only_at_start"], []string{"dns", "log-driver"}) {
t.Errorf("start-only: %v", got["read_only_at_start"])
}
c.ReadFile = func(string) ([]byte, error) { return nil, os.ErrNotExist }
got, _ = c.DaemonConfig(context.Background())
if !strings.HasPrefix(got["file_state"].(string), "absent") {
t.Errorf("absent: %v", got["file_state"])
}
c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") }
got, _ = c.DaemonConfig(context.Background())
if !strings.HasPrefix(got["file_state"].(string), "unreadable") {
t.Errorf("unreadable: %v", got["file_state"])
}
}
func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
out := `{"Type":"container","Action":"exec_start: pg_isready","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"db"}},"timeNano":1}
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
`
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
if err != nil {
t.Fatal(err)
}
evs := got["events"].([]map[string]any)
if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" {
t.Fatalf("%v", evs)
}
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
t.Fatal("an unknown type was accepted")
}
}
func TestVolumesSayWhoMountsThemAndWhetherTheMeshDoes(t *testing.T) {
f := machine().
on("docker volume ls --quiet", Ran{Stdout: "webdata\ndbdata\nloose\n"}).
on("docker volume inspect", Ran{Stdout: `[{"Name":"webdata","Driver":"local"},{"Name":"dbdata","Driver":"local"},{"Name":"loose","Driver":"local","Labels":{"com.docker.volume.anonymous":""}}]`})
got, err := client(f, 1000).Volumes(context.Background(), false, false)
if err != nil {
t.Fatal(err)
}
vols := got["volumes"].([]map[string]any)
if vols[0]["mesh_held"] != true || vols[1]["mesh_held"] != false || len(vols[2]["mounted_by"].([]map[string]any)) != 0 || vols[2]["anonymous"] != true {
t.Fatalf("%v", vols)
}
got, _ = client(f, 1000).Volumes(context.Background(), true, false)
if got["count"] != 1 {
t.Fatalf("unmounted: %v", got)
}
}
func TestImagesNameTheirUsers(t *testing.T) {
f := machine().on("docker image ls", Ran{Stdout: `{"ID":"sha256:img1","Repository":"web","Tag":"1","Size":"100MB"}
{"ID":"sha256:img3","Repository":"<none>","Tag":"<none>","Size":"2GB"}
`})
got, err := client(f, 1000).Images(context.Background(), "", "", 10)
if err != nil {
t.Fatal(err)
}
imgs := got["images"].([]Image)
if imgs[0].ID != "sha256:img3" || !imgs[0].Dangling || imgs[1].UsedBy[0] != "mesh-web" || !imgs[1].MeshUsed {
t.Fatalf("%+v", imgs)
}
got, _ = client(f, 1000).Images(context.Background(), "unused", "", 10)
if got["count"] != 1 {
t.Fatalf("unused: %v", got)
}
}
func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) {
c := client(machine(), 1000)
p, err := c.Problems(context.Background())
if err != nil || len(p) != 1 || p[0]["name"] != "dev-db" || p[0]["why"].([]string)[0] != "exited 1" {
t.Fatalf("%v %v", p, err)
}
u, err := c.Unlabelled(context.Background())
if err != nil || u["count"] != 1 {
t.Fatalf("%v %v", u, err)
}
}
// eventLine is one line of `docker events --format '{{json .}}'`.
func eventLine(at time.Time, kind, action, name string) string {
return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n",
kind, action, name, at.UnixNano())
}
// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the
// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened.
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
var out strings.Builder
for i := 0; i < 250; i++ {
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
}
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
if err != nil {
t.Fatal(err)
}
if got["count"] != 0 || got["held"] != 250 {
t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"])
}
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
t.Fatalf("an empty answer from a short history reads as quiet: %v", got)
}
}
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false)
if err != nil {
t.Fatal(err)
}
if got["complete"] != true || got["count"] != 2 || got["history"] != nil {
t.Fatalf("a covered window: %v", got)
}
if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") {
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
}
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != false || got["history"] == nil {
t.Fatalf("no events at all reads as quiet: %v", got)
}
}