From 2fde82993d651ae5bcc0c342ac3e8a9bfa1ee568 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:33:23 +0200 Subject: [PATCH] docker: read the resolver file a container really uses (hq issue 348) On 2026-10-09 the control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the machine's file listed both mesh resolvers; its copy lives under the runtime's root-only directory and no tool read it. docker_resolv_conf reads /etc/resolv.conf as the container sees it through the runtime's archive (docker cp, no exec), parses it, and says whether it lists the same resolvers as the machine. --- modules/docker/cmd/docker-tools/main.go | 13 ++ modules/docker/cmd/docker-tools/resolv.go | 111 ++++++++++++++++++ .../docker/cmd/docker-tools/resolv_test.go | 83 +++++++++++++ modules/docker/module.json | 6 + 4 files changed, 213 insertions(+) create mode 100644 modules/docker/cmd/docker-tools/resolv.go create mode 100644 modules/docker/cmd/docker-tools/resolv_test.go diff --git a/modules/docker/cmd/docker-tools/main.go b/modules/docker/cmd/docker-tools/main.go index 9e95a39d..66abc0bf 100644 --- a/modules/docker/cmd/docker-tools/main.go +++ b/modules/docker/cmd/docker-tools/main.go @@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool { return c.Inspect(ctx, ref) }, }, + { + Name: "docker_resolv_conf", + Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " + + "its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.", + Input: map[string]any{"container": containerArg}, + Run: func(args map[string]any) (any, error) { + ref, err := text(args, "container") + if err != nil { + return nil, err + } + return c.ResolvConfOf(ctx, ref) + }, + }, { Name: "docker_logs", Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " + diff --git a/modules/docker/cmd/docker-tools/resolv.go b/modules/docker/cmd/docker-tools/resolv.go new file mode 100644 index 00000000..e04750fa --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv.go @@ -0,0 +1,111 @@ +package main + +// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's +// resolver file, made by the runtime when it is created and kept under the runtime's root-only +// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the +// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the +// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the +// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream): +// no exec into the container, nothing written, and it works where the program has no shell. + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "strings" +) + +// ContainerResolvConf is the path read inside a container. +const ContainerResolvConf = "/etc/resolv.conf" + +// HostResolvConf is the machine's own resolver file, read for comparison. +const HostResolvConf = "/etc/resolv.conf" + +// ResolvConf is a resolver file, parsed. +type ResolvConf struct { + Nameservers []string `json:"nameservers"` + Search []string `json:"search,omitempty"` + Options []string `json:"options,omitempty"` + Raw string `json:"raw"` +} + +// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only. +func parseResolvConf(text string) ResolvConf { + out := ResolvConf{Nameservers: []string{}, Raw: text} + for _, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") { + continue + } + switch f[0] { + case "nameserver": + if len(f) > 1 { + out.Nameservers = append(out.Nameservers, f[1]) + } + case "search", "domain": + out.Search = append(out.Search, f[1:]...) + case "options": + out.Options = append(out.Options, f[1:]...) + } + } + return out +} + +// fileFromArchive is the one regular file a `docker cp : -` stream carries. +func fileFromArchive(stream string) (string, error) { + tr := tar.NewReader(strings.NewReader(stream)) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return "", errors.New("the runtime's archive held no file") + } + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + switch h.Typeflag { + case tar.TypeReg: + body, err := io.ReadAll(io.LimitReader(tr, 64<<10)) + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + return string(body), nil + case tar.TypeSymlink: + return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname) + } + } +} + +// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list +// the same resolvers in the same order. +func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) { + ref, err := Ref(ref) + if err != nil { + return nil, err + } + answer := map[string]any{"container": ref, "path": ContainerResolvConf} + stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-") + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + text, err := fileFromArchive(stream) + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + inside := parseResolvConf(text) + answer["resolv_conf"] = inside + if raw, err := c.ReadFile(HostResolvConf); err != nil { + answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error() + } else { + host := parseResolvConf(string(raw)) + answer["machine_nameservers"] = host.Nameservers + same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ") + answer["same_as_machine"] = same + if !same { + answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " + + "or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again" + } + } + return answer, nil +} diff --git a/modules/docker/cmd/docker-tools/resolv_test.go b/modules/docker/cmd/docker-tools/resolv_test.go new file mode 100644 index 00000000..7512c7c3 --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func archiveOf(t *testing.T, h tar.Header, body string) string { + t.Helper() + var buf bytes.Buffer + w := tar.NewWriter(&buf) + h.Size = int64(len(body)) + if h.Typeflag == tar.TypeSymlink { + h.Size = 0 + } + if err := w.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if h.Typeflag != tar.TypeSymlink { + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + w.Close() + return buf.String() +} + +// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy +// of the file could not be read by anything on the mesh. +func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) { + inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n" + f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -", + Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)}) + c := client(f, 1000) + c.ReadFile = func(p string) ([]byte, error) { + if p != HostResolvConf { + t.Fatalf("read %s", p) + } + return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil + } + got, err := c.ResolvConfOf(context.Background(), "route-proxy") + if err != nil { + t.Fatal(err) + } + rc := got["resolv_conf"].(ResolvConf) + if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" || + strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside { + t.Fatalf("parsed: %+v", rc) + } + if got["same_as_machine"] != false || got["note"] == nil { + t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got) + } + if f.ran("docker exec") { + t.Fatal("read by exec, not through the archive") + } +} + +func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) { + c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Fatalf("a missing container: %v", err) + } + link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "") + c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") { + t.Fatalf("a link inside: %v", err) + } + c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil { + t.Fatal("garbage was read as a file") + } + ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n") + c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000) + c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") } + got, err := c.ResolvConfOf(context.Background(), "x") + if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil { + t.Fatalf("an unreadable machine file: %v, %v", got, err) + } +} diff --git a/modules/docker/module.json b/modules/docker/module.json index b7fc0726..73c8f74c 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -18,6 +18,7 @@ "tools": [ "docker_list", "docker_inspect", + "docker_resolv_conf", "docker_logs", "docker_secrets_in_logs", "docker_secrets_in_events", @@ -38,6 +39,11 @@ "docker_problems", "docker_ports" ], + "replaces": { + "docker_resolv_conf": [ + "docker exec cat /etc/resolv.conf" + ] + }, "resources": [ { "id": "package",