docker: read the resolver file a container really uses (hq issue 348)

On 2026-10-09 the control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty
minutes while the machine's file listed both mesh resolvers; its copy lives under the runtime's
root-only directory and no tool read it. docker_resolv_conf reads /etc/resolv.conf as the container
sees it through the runtime's archive (docker cp, no exec), parses it, and says whether it lists the
same resolvers as the machine.

modules/docker/cmd/docker-tools/main.go

@@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool {
 				return c.Inspect(ctx, ref)
 			},
 		},
+		{
+			Name: "docker_resolv_conf",
+			Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " +
+				"its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.",
+			Input: map[string]any{"container": containerArg},
+			Run: func(args map[string]any) (any, error) {
+				ref, err := text(args, "container")
+				if err != nil {
+					return nil, err
+				}
+				return c.ResolvConfOf(ctx, ref)
+			},
+		},
 		{
 			Name: "docker_logs",
 			Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " +

modules/docker/cmd/docker-tools/resolv.go

@@ -0,0 +1,111 @@
+package main
+
+// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's
+// resolver file, made by the runtime when it is created and kept under the runtime's root-only
+// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the
+// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the
+// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the
+// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream):
+// no exec into the container, nothing written, and it works where the program has no shell.
+
+import (
+	"archive/tar"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// ContainerResolvConf is the path read inside a container.
+const ContainerResolvConf = "/etc/resolv.conf"
+
+// HostResolvConf is the machine's own resolver file, read for comparison.
+const HostResolvConf = "/etc/resolv.conf"
+
+// ResolvConf is a resolver file, parsed.
+type ResolvConf struct {
+	Nameservers []string `json:"nameservers"`
+	Search      []string `json:"search,omitempty"`
+	Options     []string `json:"options,omitempty"`
+	Raw         string   `json:"raw"`
+}
+
+// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only.
+func parseResolvConf(text string) ResolvConf {
+	out := ResolvConf{Nameservers: []string{}, Raw: text}
+	for _, line := range strings.Split(text, "\n") {
+		f := strings.Fields(line)
+		if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") {
+			continue
+		}
+		switch f[0] {
+		case "nameserver":
+			if len(f) > 1 {
+				out.Nameservers = append(out.Nameservers, f[1])
+			}
+		case "search", "domain":
+			out.Search = append(out.Search, f[1:]...)
+		case "options":
+			out.Options = append(out.Options, f[1:]...)
+		}
+	}
+	return out
+}
+
+// fileFromArchive is the one regular file a `docker cp <ref>:<path> -` stream carries.
+func fileFromArchive(stream string) (string, error) {
+	tr := tar.NewReader(strings.NewReader(stream))
+	for {
+		h, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return "", errors.New("the runtime's archive held no file")
+		}
+		if err != nil {
+			return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
+		}
+		switch h.Typeflag {
+		case tar.TypeReg:
+			body, err := io.ReadAll(io.LimitReader(tr, 64<<10))
+			if err != nil {
+				return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
+			}
+			return string(body), nil
+		case tar.TypeSymlink:
+			return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname)
+		}
+	}
+}
+
+// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list
+// the same resolvers in the same order.
+func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) {
+	ref, err := Ref(ref)
+	if err != nil {
+		return nil, err
+	}
+	answer := map[string]any{"container": ref, "path": ContainerResolvConf}
+	stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-")
+	if err != nil {
+		return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
+	}
+	text, err := fileFromArchive(stream)
+	if err != nil {
+		return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
+	}
+	inside := parseResolvConf(text)
+	answer["resolv_conf"] = inside
+	if raw, err := c.ReadFile(HostResolvConf); err != nil {
+		answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error()
+	} else {
+		host := parseResolvConf(string(raw))
+		answer["machine_nameservers"] = host.Nameservers
+		same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ")
+		answer["same_as_machine"] = same
+		if !same {
+			answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " +
+				"or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again"
+		}
+	}
+	return answer, nil
+}

modules/docker/cmd/docker-tools/resolv_test.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"errors"
+	"strings"
+	"testing"
+)
+
+func archiveOf(t *testing.T, h tar.Header, body string) string {
+	t.Helper()
+	var buf bytes.Buffer
+	w := tar.NewWriter(&buf)
+	h.Size = int64(len(body))
+	if h.Typeflag == tar.TypeSymlink {
+		h.Size = 0
+	}
+	if err := w.WriteHeader(&h); err != nil {
+		t.Fatal(err)
+	}
+	if h.Typeflag != tar.TypeSymlink {
+		if _, err := w.Write([]byte(body)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	w.Close()
+	return buf.String()
+}
+
+// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy
+// of the file could not be read by anything on the mesh.
+func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) {
+	inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n"
+	f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -",
+		Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)})
+	c := client(f, 1000)
+	c.ReadFile = func(p string) ([]byte, error) {
+		if p != HostResolvConf {
+			t.Fatalf("read %s", p)
+		}
+		return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil
+	}
+	got, err := c.ResolvConfOf(context.Background(), "route-proxy")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rc := got["resolv_conf"].(ResolvConf)
+	if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" ||
+		strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside {
+		t.Fatalf("parsed: %+v", rc)
+	}
+	if got["same_as_machine"] != false || got["note"] == nil {
+		t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got)
+	}
+	if f.ran("docker exec") {
+		t.Fatal("read by exec, not through the archive")
+	}
+}
+
+func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
+	c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000)
+	if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") {
+		t.Fatalf("a missing container: %v", err)
+	}
+	link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "")
+	c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000)
+	if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") {
+		t.Fatalf("a link inside: %v", err)
+	}
+	c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000)
+	if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil {
+		t.Fatal("garbage was read as a file")
+	}
+	ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n")
+	c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000)
+	c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") }
+	got, err := c.ResolvConfOf(context.Background(), "x")
+	if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil {
+		t.Fatalf("an unreadable machine file: %v, %v", got, err)
+	}
+}

modules/docker/module.json

@@ -18,6 +18,7 @@
   "tools": [
     "docker_list",
     "docker_inspect",
+    "docker_resolv_conf",
     "docker_logs",
     "docker_secrets_in_logs",
     "docker_secrets_in_events",
@@ -38,6 +39,11 @@
     "docker_problems",
     "docker_ports"
   ],
+  "replaces": {
+    "docker_resolv_conf": [
+      "docker exec cat /etc/resolv.conf"
+    ]
+  },
   "resources": [
     {
       "id": "package",