Author SHA1 Message Date
jochen 2fde82993d docker: read the resolver file a container really uses (hq issue 348)
On 2026-10-09 the control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty
minutes while the machine's file listed both mesh resolvers; its copy lives under the runtime's
root-only directory and no tool read it. docker_resolv_conf reads /etc/resolv.conf as the container
sees it through the runtime's archive (docker cp, no exec), parses it, and says whether it lists the
same resolvers as the machine.
2026-10-09 13:33:23 +02:00
jochen c0fa282548 netcheck: ask one named resolver, and list what listens here (hq issue 348)
During the 2026-10-09 outage the control node's resolver stayed active while it no longer listened on
its private address, and nothing on the mesh could ask that address a question or show which sockets
were bound. netcheck_dns takes server (and tcp) to ask one resolver alone and say its response code;
netcheck_listening reads the kernel's socket tables, as ss -lunt does, with the owning program where
the tool's account may see it and a sentence where it may not.
2026-10-09 13:27:39 +02:00
jochen d79a4eeab9 docker: say where the runtime's event history begins, so a short history never reads as a quiet one (hq issue 348)
docker_events asked for three hours across the 2026-10-09 resolver outage on the control node answered
zero events: the runtime keeps only its last ~256 events in memory, and health-check execs filled them
with the last ten minutes. The answer now carries held, oldest_held and complete, and a history line
when the window reaches further back than what the runtime still holds. The type filter moves into the
tool, since the history's start is only known from all of it.
2026-10-09 13:24:58 +02:00
13 changed files with 1026 additions and 13 deletions
+47 -3
View File
@@ -991,16 +991,27 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
}
// Events is what the runtime did in a window ending now, the latest last.
//
// **The runtime keeps only its last events in memory** (about 256), and the images' own health checks
// make three exec events each per look: on a machine with a few checked containers that history is ten
// minutes long or less, all of it exec events (hq research 032). A window asked for further back than
// the history reaches is answered with what is still held AND says where the history begins, so an
// empty list never reads as "nothing happened" (hq issue 348: three hours asked across an outage,
// answered zero). The type filter is applied here rather than by the runtime, for the same reason: the
// history's start is only known from all of it.
func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
return c.eventsAt(ctx, time.Now(), minutes, kind, limit, execs)
}
func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
if kind != "" {
switch kind {
case "container", "image", "network", "volume", "daemon", "plugin", "builder":
default:
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
}
args = append(args, "--filter", "type="+kind)
}
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
out, err := c.docker(ctx, args...)
if err != nil {
return nil, err
@@ -1015,6 +1026,9 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int
redacted := map[string]bool{}
events := []map[string]any{}
for _, e := range raw {
if kind != "" && e.Type != kind {
continue
}
if !execs && strings.HasPrefix(e.Action, "exec_") {
continue
}
@@ -1042,14 +1056,44 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int
if len(events) > limit {
events = events[len(events)-limit:]
}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
"held": len(raw)}
if len(redacted) > 0 {
answer["redacted"] = sortedSet(redacted)
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
}
for k, v := range historyReach(raw, now, minutes) {
answer[k] = v
}
return answer, nil
}
// historyGap is how much later than the window's start the runtime's oldest held event may be and the
// window still read as covered: events are not made every second.
const historyGap = 2 * time.Minute
// historyReach says where the runtime's history begins when it begins after the window asked for: the
// oldest event it still holds, and that nothing before it can be answered from here.
func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any {
start := now.Add(-time.Duration(minutes) * time.Minute)
if len(raw) == 0 {
return map[string]any{"complete": false,
"history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"}
}
oldest := time.Unix(0, raw[0].TimeNano)
for _, e := range raw {
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
oldest = t
}
}
if oldest.Sub(start) <= historyGap {
return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)}
}
return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339),
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
}
func sortedSet(set map[string]bool) []string {
out := make([]string, 0, len(set))
for k := range set {
+52 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"reflect"
@@ -307,7 +308,8 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" {
t.Fatalf("%v", evs)
}
if !f.ran("docker events --since 30m --until 0s --format {{json .}} --filter type=container") {
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
@@ -362,3 +364,52 @@ func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) {
t.Fatalf("%v %v", u, err)
}
}
// eventLine is one line of `docker events --format '{{json .}}'`.
func eventLine(at time.Time, kind, action, name string) string {
return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n",
kind, action, name, at.UnixNano())
}
// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the
// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened.
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
var out strings.Builder
for i := 0; i < 250; i++ {
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
}
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
if err != nil {
t.Fatal(err)
}
if got["count"] != 0 || got["held"] != 250 {
t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"])
}
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
t.Fatalf("an empty answer from a short history reads as quiet: %v", got)
}
}
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false)
if err != nil {
t.Fatal(err)
}
if got["complete"] != true || got["count"] != 2 || got["history"] != nil {
t.Fatalf("a covered window: %v", got)
}
if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") {
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
}
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != false || got["history"] == nil {
t.Fatalf("no events at all reads as quiet: %v", got)
}
}
+15 -1
View File
@@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool {
return c.Inspect(ctx, ref)
},
},
{
Name: "docker_resolv_conf",
Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " +
"its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.",
Input: map[string]any{"container": containerArg},
Run: func(args map[string]any) (any, error) {
ref, err := text(args, "container")
if err != nil {
return nil, err
}
return c.ResolvConfOf(ctx, ref)
},
},
{
Name: "docker_logs",
Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " +
@@ -262,7 +275,8 @@ func tools(c *Client) []stdio.Tool {
{
Name: "docker_events",
Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " +
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag.",
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag. " +
"The runtime keeps only its last ~256 events in memory, which health-check execs fill in minutes: complete=false and history say where what it holds begins, and an empty list then means gone, not quiet.",
Input: map[string]any{
"minutes": map[string]any{"type": "integer", "description": "how far back (default 60, at most 1440)"},
"type": map[string]any{"type": "string", "description": "only one kind: container, image, network, volume, daemon, plugin or builder"},
+111
View File
@@ -0,0 +1,111 @@
package main
// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's
// resolver file, made by the runtime when it is created and kept under the runtime's root-only
// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the
// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the
// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the
// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream):
// no exec into the container, nothing written, and it works where the program has no shell.
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"strings"
)
// ContainerResolvConf is the path read inside a container.
const ContainerResolvConf = "/etc/resolv.conf"
// HostResolvConf is the machine's own resolver file, read for comparison.
const HostResolvConf = "/etc/resolv.conf"
// ResolvConf is a resolver file, parsed.
type ResolvConf struct {
Nameservers []string `json:"nameservers"`
Search []string `json:"search,omitempty"`
Options []string `json:"options,omitempty"`
Raw string `json:"raw"`
}
// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only.
func parseResolvConf(text string) ResolvConf {
out := ResolvConf{Nameservers: []string{}, Raw: text}
for _, line := range strings.Split(text, "\n") {
f := strings.Fields(line)
if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") {
continue
}
switch f[0] {
case "nameserver":
if len(f) > 1 {
out.Nameservers = append(out.Nameservers, f[1])
}
case "search", "domain":
out.Search = append(out.Search, f[1:]...)
case "options":
out.Options = append(out.Options, f[1:]...)
}
}
return out
}
// fileFromArchive is the one regular file a `docker cp <ref>:<path> -` stream carries.
func fileFromArchive(stream string) (string, error) {
tr := tar.NewReader(strings.NewReader(stream))
for {
h, err := tr.Next()
if errors.Is(err, io.EOF) {
return "", errors.New("the runtime's archive held no file")
}
if err != nil {
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
}
switch h.Typeflag {
case tar.TypeReg:
body, err := io.ReadAll(io.LimitReader(tr, 64<<10))
if err != nil {
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
}
return string(body), nil
case tar.TypeSymlink:
return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname)
}
}
}
// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list
// the same resolvers in the same order.
func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) {
ref, err := Ref(ref)
if err != nil {
return nil, err
}
answer := map[string]any{"container": ref, "path": ContainerResolvConf}
stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-")
if err != nil {
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
}
text, err := fileFromArchive(stream)
if err != nil {
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
}
inside := parseResolvConf(text)
answer["resolv_conf"] = inside
if raw, err := c.ReadFile(HostResolvConf); err != nil {
answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error()
} else {
host := parseResolvConf(string(raw))
answer["machine_nameservers"] = host.Nameservers
same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ")
answer["same_as_machine"] = same
if !same {
answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " +
"or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again"
}
}
return answer, nil
}
@@ -0,0 +1,83 @@
package main
import (
"archive/tar"
"bytes"
"context"
"errors"
"strings"
"testing"
)
func archiveOf(t *testing.T, h tar.Header, body string) string {
t.Helper()
var buf bytes.Buffer
w := tar.NewWriter(&buf)
h.Size = int64(len(body))
if h.Typeflag == tar.TypeSymlink {
h.Size = 0
}
if err := w.WriteHeader(&h); err != nil {
t.Fatal(err)
}
if h.Typeflag != tar.TypeSymlink {
if _, err := w.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
w.Close()
return buf.String()
}
// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy
// of the file could not be read by anything on the mesh.
func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) {
inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n"
f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -",
Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)})
c := client(f, 1000)
c.ReadFile = func(p string) ([]byte, error) {
if p != HostResolvConf {
t.Fatalf("read %s", p)
}
return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil
}
got, err := c.ResolvConfOf(context.Background(), "route-proxy")
if err != nil {
t.Fatal(err)
}
rc := got["resolv_conf"].(ResolvConf)
if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" ||
strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside {
t.Fatalf("parsed: %+v", rc)
}
if got["same_as_machine"] != false || got["note"] == nil {
t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got)
}
if f.ran("docker exec") {
t.Fatal("read by exec, not through the archive")
}
}
func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") {
t.Fatalf("a missing container: %v", err)
}
link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "")
c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") {
t.Fatalf("a link inside: %v", err)
}
c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil {
t.Fatal("garbage was read as a file")
}
ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n")
c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000)
c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") }
got, err := c.ResolvConfOf(context.Background(), "x")
if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil {
t.Fatalf("an unreadable machine file: %v, %v", got, err)
}
}
+6
View File
@@ -18,6 +18,7 @@
"tools": [
"docker_list",
"docker_inspect",
"docker_resolv_conf",
"docker_logs",
"docker_secrets_in_logs",
"docker_secrets_in_events",
@@ -38,6 +39,11 @@
"docker_problems",
"docker_ports"
],
"replaces": {
"docker_resolv_conf": [
"docker exec cat /etc/resolv.conf"
]
},
"resources": [
{
"id": "package",
+252
View File
@@ -0,0 +1,252 @@
package main
// Asking one named resolver directly (hq issue 348). netcheck_dns without a server asks the machine's
// resolver the way its programs do — resolv.conf, hosts, search domains, several servers in turn — which
// answers "can this machine resolve?" and hides which resolver answered. During the 2026-10-09 outage the
// question was "does the resolver at 10.10.0.1 answer?", and nothing on the mesh could ask it. Asked with a
// server, the question goes to that address and port alone, over UDP or TCP, and the answer says its
// response code: a refusal, no such name, and no answer at all are three different findings.
//
// The wire format is written here rather than taken from a library: a question and the records of five
// types are all it needs, and the module stays without a dependency.
import (
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
)
var dnsTypeCodes = map[string]uint16{"A": 1, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28}
var rcodeWords = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
// serverAddress is a server as asked — an address, with a port or without (53) — as host:port.
func serverAddress(server string) (string, error) {
server = strings.TrimSpace(server)
if host, port, err := net.SplitHostPort(server); err == nil {
p, err := strconv.Atoi(port)
if err != nil || p < 1 || p > 65535 {
return "", fmt.Errorf("server %q: the port is not one (1-65535)", server)
}
if net.ParseIP(host) == nil {
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
}
return net.JoinHostPort(host, port), nil
}
if net.ParseIP(server) == nil {
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
}
return net.JoinHostPort(server, "53"), nil
}
// askServer asks one resolver one question and reads its answer.
func askServer(server, name, kind string, overTCP bool, timeout time.Duration) (rcode int, answers []string, err error) {
addr, err := serverAddress(server)
if err != nil {
return 0, nil, err
}
query, id, err := buildQuery(name, dnsTypeCodes[kind])
if err != nil {
return 0, nil, err
}
network := "udp"
if overTCP {
network = "tcp"
}
conn, err := net.DialTimeout(network, addr, timeout)
if err != nil {
return 0, nil, err
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(timeout))
var reply []byte
if overTCP {
framed := binary.BigEndian.AppendUint16(nil, uint16(len(query)))
if _, err := conn.Write(append(framed, query...)); err != nil {
return 0, nil, err
}
var size [2]byte
if _, err := readFull(conn, size[:]); err != nil {
return 0, nil, err
}
reply = make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := readFull(conn, reply); err != nil {
return 0, nil, err
}
} else {
if _, err := conn.Write(query); err != nil {
return 0, nil, err
}
buf := make([]byte, 4096)
n, err := conn.Read(buf)
if err != nil {
return 0, nil, err
}
reply = buf[:n]
}
return parseReply(reply, id, dnsTypeCodes[kind])
}
func readFull(conn net.Conn, b []byte) (int, error) {
got := 0
for got < len(b) {
n, err := conn.Read(b[got:])
got += n
if err != nil {
return got, err
}
}
return got, nil
}
// buildQuery is one question, recursion desired, under a random id.
func buildQuery(name string, qtype uint16) ([]byte, uint16, error) {
var idb [2]byte
_, _ = rand.Read(idb[:])
id := binary.BigEndian.Uint16(idb[:])
msg := binary.BigEndian.AppendUint16(nil, id)
msg = append(msg, 0x01, 0x00) // recursion desired
msg = append(msg, 0, 1, 0, 0, 0, 0, 0, 0) // one question
wire, err := encodeName(name)
if err != nil {
return nil, 0, err
}
msg = append(msg, wire...)
msg = binary.BigEndian.AppendUint16(msg, qtype)
msg = binary.BigEndian.AppendUint16(msg, 1) // IN
return msg, id, nil
}
func encodeName(name string) ([]byte, error) {
name = strings.TrimSuffix(strings.TrimSpace(name), ".")
var out []byte
if name != "" {
for _, label := range strings.Split(name, ".") {
if label == "" || len(label) > 63 {
return nil, fmt.Errorf("name %q has an empty or too long label", name)
}
out = append(out, byte(len(label)))
out = append(out, label...)
}
}
return append(out, 0), nil
}
var errShort = errors.New("the answer is cut short")
// readName reads a possibly compressed name at off, and the offset after it.
func readName(msg []byte, off int) (string, int, error) {
var labels []string
next, jumped := -1, 0
for {
if off >= len(msg) {
return "", 0, errShort
}
l := int(msg[off])
switch {
case l == 0:
if next < 0 {
next = off + 1
}
return strings.Join(labels, "."), next, nil
case l&0xC0 == 0xC0:
if off+1 >= len(msg) {
return "", 0, errShort
}
if next < 0 {
next = off + 2
}
if jumped++; jumped > 32 {
return "", 0, errors.New("the answer's names point in a loop")
}
off = int(binary.BigEndian.Uint16(msg[off:]) & 0x3FFF)
default:
if off+1+l > len(msg) {
return "", 0, errShort
}
labels = append(labels, string(msg[off+1:off+1+l]))
off += 1 + l
}
}
}
// parseReply reads the response code and the answers of the asked type.
func parseReply(msg []byte, id, qtype uint16) (int, []string, error) {
if len(msg) < 12 {
return 0, nil, errShort
}
if binary.BigEndian.Uint16(msg) != id {
return 0, nil, errors.New("the answer is to another question (its id differs)")
}
if msg[2]&0x80 == 0 {
return 0, nil, errors.New("the answer is not marked as one")
}
rcode := int(msg[3] & 0x0F)
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
off := 12
for i := 0; i < qd; i++ {
_, n, err := readName(msg, off)
if err != nil {
return rcode, nil, err
}
off = n + 4
}
answers := []string{}
for i := 0; i < an; i++ {
_, n, err := readName(msg, off)
if err != nil {
return rcode, answers, err
}
off = n
if off+10 > len(msg) {
return rcode, answers, errShort
}
rtype := binary.BigEndian.Uint16(msg[off:])
size := int(binary.BigEndian.Uint16(msg[off+8:]))
data := off + 10
if data+size > len(msg) {
return rcode, answers, errShort
}
rdata := msg[data : data+size]
off = data + size
if rtype != qtype {
continue // a CNAME on the way to an address, say
}
switch rtype {
case 1, 28:
answers = append(answers, net.IP(rdata).String())
case 5:
s, _, err := readName(msg, data)
if err != nil {
return rcode, answers, err
}
answers = append(answers, s)
case 15:
if size < 3 {
return rcode, answers, errShort
}
s, _, err := readName(msg, data+2)
if err != nil {
return rcode, answers, err
}
answers = append(answers, fmt.Sprintf("%d %s", binary.BigEndian.Uint16(rdata), s))
case 16:
var parts []string
for p := 0; p < len(rdata); {
l := int(rdata[p])
if p+1+l > len(rdata) {
return rcode, answers, errShort
}
parts = append(parts, string(rdata[p+1:p+1+l]))
p += 1 + l
}
answers = append(answers, strings.Join(parts, ""))
}
}
return rcode, answers, nil
}
+175
View File
@@ -0,0 +1,175 @@
package main
import (
"encoding/binary"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode.
func respond(q []byte, rcode byte) []byte {
end := 12
for q[end] != 0 {
end += int(q[end]) + 1
}
question := q[12 : end+5]
name := strings.ToLower(string(q[13:end]))
r := append([]byte{}, q[:2]...)
r = append(r, 0x81, 0x80|rcode, 0, 1)
answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1
if answer {
r = append(r, 0, 1, 0, 0, 0, 0)
} else {
r = append(r, 0, 0, 0, 0, 0, 0)
}
r = append(r, question...)
if answer {
r = append(r, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1)
}
return r
}
func udpResolver(t *testing.T, rcode byte) string {
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { pc.Close() })
go func() {
buf := make([]byte, 512)
for {
n, from, err := pc.ReadFrom(buf)
if err != nil {
return
}
_, _ = pc.WriteTo(respond(buf[:n], rcode), from)
}
}()
return pc.LocalAddr().String()
}
func tcpResolver(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { l.Close() })
go func() {
for {
c, err := l.Accept()
if err != nil {
return
}
var size [2]byte
if _, err := readFull(c, size[:]); err == nil {
q := make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := readFull(c, q); err == nil {
r := respond(q, 0)
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
}
}
c.Close()
}
}()
return l.Addr().String()
}
// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer?
func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) {
got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" {
t.Fatalf("udp: %+v, %v", got, err)
}
got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" {
t.Fatalf("tcp: %+v, %v", got, err)
}
got, _ = AskDNS("nope.internal", "A", udpResolver(t, 3), false, time.Second)
if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 {
t.Fatalf("no such name is a result with its code: %+v", got)
}
got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second)
if got.Rcode != "REFUSED" || got.Error == "" {
t.Fatalf("a refusal: %+v", got)
}
}
func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) {
l, _ := net.Listen("tcp", "127.0.0.1:0")
addr := l.Addr().String()
l.Close()
got, err := AskDNS("novox.internal", "A", addr, true, time.Second)
if err != nil || !strings.Contains(got.Error, "connection refused") {
t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err)
}
if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil {
t.Fatal("a server given by name was accepted")
}
if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil {
t.Fatal("tcp without a server was accepted")
}
if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" {
t.Fatalf("default port: %s", a)
}
}
func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) {
root := t.TempDir()
must := func(err error) {
if err != nil {
t.Fatal(err)
}
}
must(os.MkdirAll(filepath.Join(root, "net"), 0o755))
head := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"
// 127.0.0.1:53 and 10.10.0.1:53 listening, a connection on 22 (not listening), and an udp 53 bound on loopback only.
must(os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(head+
" 0: 0100007F:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 111 1 0 100 0 0 10 0\n"+
" 1: 01000A0A:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 112 1 0 100 0 0 10 0\n"+
" 2: 0100007F:0016 0200007F:9C40 01 00000000:00000000 00:00000000 00000000 0 0 113 1 0 100 0 0 10 0\n"), 0o644))
must(os.WriteFile(filepath.Join(root, "net", "udp"), []byte(head+
" 0: 0100007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000 0 0 121 2 0 0\n"), 0o644))
must(os.MkdirAll(filepath.Join(root, "42", "fd"), 0o755))
must(os.WriteFile(filepath.Join(root, "42", "comm"), []byte("dnsmasq\n"), 0o644))
must(os.Symlink("socket:[111]", filepath.Join(root, "42", "fd", "5")))
got := Listening(root, 53)
if got.Count != 3 || len(got.Unread) != 0 {
t.Fatalf("%+v", got)
}
var where []string
for _, s := range got.Sockets {
where = append(where, s.Protocol+" "+s.Address)
}
if strings.Join(where, ",") != "tcp 10.10.0.1,tcp 127.0.0.1,udp 127.0.0.1" {
t.Fatalf("sockets: %v", where)
}
if got.Sockets[1].Process != "42/dnsmasq" || got.Sockets[0].Owner == "" || !strings.Contains(got.Owners, "2 socket(s)") {
t.Fatalf("owners: %+v", got)
}
if missing := Listening(t.TempDir(), 0); len(missing.Unread) != 2 {
t.Fatalf("an unreadable table is said, not left out: %+v", missing)
}
}
func TestListeningOnThisMachineFindsASocketItOpened(t *testing.T) {
if _, err := os.Stat("/proc/net/tcp"); err != nil {
t.Skip("no /proc here")
}
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
port := l.Addr().(*net.TCPAddr).Port
got := Listening("/proc", port)
if got.Count != 1 || got.Sockets[0].Address != "127.0.0.1" {
t.Fatalf("%+v", got)
}
if got.Sockets[0].Process == "" {
t.Fatalf("its own socket's program is not named: %+v", got)
}
}
+58 -2
View File
@@ -50,8 +50,13 @@ func CheckTCP(host string, port int, timeout time.Duration) (TCPResult, error) {
// DNSResult is what netcheck_dns answers.
type DNSResult struct {
Name string `json:"name"`
Type string `json:"type"`
Name string `json:"name"`
Type string `json:"type"`
// Server and Protocol say which resolver was asked when one was named; empty, the machine's.
Server string `json:"server,omitempty"`
Protocol string `json:"protocol,omitempty"`
// Rcode is the named resolver's response code: NOERROR, NXDOMAIN, REFUSED, SERVFAIL…
Rcode string `json:"rcode,omitempty"`
Answers []string `json:"answers"`
ElapsedMS int64 `json:"elapsed_ms"`
Error string `json:"error,omitempty"`
@@ -64,6 +69,13 @@ var DNSTypes = []string{"A", "AAAA", "CNAME", "TXT", "MX"}
// the machine's /etc/resolv.conf and /etc/hosts, which is the resolver this machine's programs use.
// A name that does not resolve is a result with its error; an unknown type is refused.
func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
return AskDNS(name, kind, "", false, timeout)
}
// AskDNS is CheckDNS, or — with a server — the same question put to that resolver alone, over UDP or
// TCP, its hosts file and search domains left out (hq issue 348). No answer, a refusal and a response
// code other than NOERROR are results with their error; a malformed question is refused.
func AskDNS(name, kind, server string, overTCP bool, timeout time.Duration) (DNSResult, error) {
kind = strings.ToUpper(strings.TrimSpace(kind))
if kind == "" {
kind = "A"
@@ -76,6 +88,37 @@ func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
return DNSResult{}, fmt.Errorf("type %q is not one netcheck_dns looks up (%s)", kind, strings.Join(DNSTypes, ", "))
}
out := DNSResult{Name: name, Type: kind, Answers: []string{}}
if server != "" {
if _, err := serverAddress(server); err != nil {
return DNSResult{}, err
}
out.Server, out.Protocol = server, "udp"
if overTCP {
out.Protocol = "tcp"
}
start := time.Now()
rcode, answers, err := askServer(server, name, kind, overTCP, timeout)
out.ElapsedMS = time.Since(start).Milliseconds()
if err == nil {
out.Rcode = rcodeWords[rcode]
if out.Rcode == "" {
out.Rcode = "RCODE" + strconv.Itoa(rcode)
}
if rcode != 0 {
out.Error = "the resolver answered " + out.Rcode
}
}
out.Answers = append(out.Answers, answers...)
if err != nil {
out.Error = dnsWords(err, timeout)
}
if kind != "MX" {
sort.Strings(out.Answers)
}
return out, nil
} else if overTCP {
return DNSResult{}, errors.New("tcp asks a named server; give server too")
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
r := net.DefaultResolver
@@ -171,3 +214,16 @@ func timeoutOf(args map[string]any) (time.Duration, error) {
}
return d, nil
}
// dnsWords is a failed question in words: refused (nothing listens there), no answer in time, or what the
// system said.
func dnsWords(err error, timeout time.Duration) string {
var ne net.Error
switch {
case strings.Contains(err.Error(), "connection refused"):
return "connection refused: nothing listens on that address and port"
case errors.As(err, &ne) && ne.Timeout():
return fmt.Sprintf("no answer within %s", timeout)
}
return err.Error()
}
+2 -2
View File
@@ -57,12 +57,12 @@ func TestTimeoutIsDefaultedAndBounded(t *testing.T) {
}
}
func TestBothToolsAreListedUnprefixed(t *testing.T) {
func TestTheToolsAreListedUnprefixed(t *testing.T) {
names := map[string]bool{}
for _, tool := range tools() {
names[tool.Name] = true
}
if !names["netcheck_tcp"] || !names["netcheck_dns"] || len(names) != 2 {
if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 {
t.Fatalf("tools: %v", names)
}
}
+179
View File
@@ -0,0 +1,179 @@
package main
// What listens on this machine (hq issue 348): the kernel's own socket tables, read where `ss -lunt` reads
// them. During the 2026-10-09 outage the resolver's unit stayed active while it no longer listened on its
// private address, and nothing on the mesh could show that: whether a socket is bound is the kernel's to
// say, not the unit's.
import (
"bufio"
"encoding/hex"
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// Socket is one listening TCP socket or one bound UDP socket.
type Socket struct {
Protocol string `json:"protocol"`
Address string `json:"address"`
Port int `json:"port"`
// Process is the program holding it, "pid/name"; empty when Owner says why it is not known.
Process string `json:"process,omitempty"`
Owner string `json:"owner,omitempty"`
inode string
}
// ListeningResult is what netcheck_listening answers.
type ListeningResult struct {
Sockets []Socket `json:"sockets"`
Count int `json:"count"`
// Unread names a table that could not be read: a socket in it is missing from the list, not absent.
Unread []string `json:"unread,omitempty"`
// Owners says how many sockets' programs are unknown and why, when any is.
Owners string `json:"owners,omitempty"`
}
// Listening reads the socket tables under root (the machine's /proc), keeps TCP sockets in the listen
// state and every bound UDP socket, on port when port is not 0.
func Listening(root string, port int) ListeningResult {
out := ListeningResult{Sockets: []Socket{}}
for _, t := range []struct{ file, protocol string }{
{"tcp", "tcp"}, {"tcp6", "tcp6"}, {"udp", "udp"}, {"udp6", "udp6"},
} {
f, err := os.Open(filepath.Join(root, "net", t.file))
if err != nil {
if !os.IsNotExist(err) || !strings.HasSuffix(t.file, "6") {
out.Unread = append(out.Unread, t.file+": "+err.Error())
}
continue
}
sockets, err := parseSocketTable(f, t.protocol, port)
f.Close()
if err != nil {
out.Unread = append(out.Unread, t.file+": "+err.Error())
}
out.Sockets = append(out.Sockets, sockets...)
}
unknown := owners(root, out.Sockets)
if unknown > 0 {
out.Owners = fmt.Sprintf("%d socket(s) without their program: held by a process of another account, whose "+
"open files this tool may not read; the address and port are the kernel's and complete", unknown)
}
sort.Slice(out.Sockets, func(i, j int) bool {
a, b := out.Sockets[i], out.Sockets[j]
if a.Port != b.Port {
return a.Port < b.Port
}
if a.Protocol != b.Protocol {
return a.Protocol < b.Protocol
}
return a.Address < b.Address
})
out.Count = len(out.Sockets)
return out
}
// parseSocketTable reads one /proc/net table: "sl local_address rem_address st … inode".
func parseSocketTable(f *os.File, protocol string, port int) ([]Socket, error) {
var out []Socket
sc := bufio.NewScanner(f)
first := true
for sc.Scan() {
if first {
first = false
continue
}
fields := strings.Fields(sc.Text())
if len(fields) < 10 {
continue
}
state := fields[3]
tcp := strings.HasPrefix(protocol, "tcp")
// TCP 0A is LISTEN; UDP 07 is a bound socket (unconnected), 01 a connected one, which is not listening.
if (tcp && state != "0A") || (!tcp && state != "07") {
continue
}
addr, p, err := hexAddress(fields[1])
if err != nil {
return out, err
}
if port != 0 && p != port {
continue
}
out = append(out, Socket{Protocol: protocol, Address: addr, Port: p, inode: fields[9]})
}
return out, sc.Err()
}
// hexAddress reads the kernel's "0100007F:0035": the address in host order per 32-bit word, the port big-endian.
func hexAddress(s string) (string, int, error) {
a, p, ok := strings.Cut(s, ":")
if !ok {
return "", 0, fmt.Errorf("address %q", s)
}
port, err := strconv.ParseUint(p, 16, 16)
if err != nil {
return "", 0, fmt.Errorf("port %q", p)
}
raw, err := hex.DecodeString(a)
if err != nil || (len(raw) != 4 && len(raw) != 16) {
return "", 0, fmt.Errorf("address %q", a)
}
ip := make(net.IP, len(raw))
for w := 0; w < len(raw); w += 4 {
ip[w], ip[w+1], ip[w+2], ip[w+3] = raw[w+3], raw[w+2], raw[w+1], raw[w]
}
return ip.String(), int(port), nil
}
// owners names each socket's program from the processes' open files, as far as they can be read, and
// counts those it could not name.
func owners(root string, sockets []Socket) int {
want := map[string][]int{}
for i, s := range sockets {
want["socket:["+s.inode+"]"] = append(want["socket:["+s.inode+"]"], i)
}
if len(want) == 0 {
return 0
}
procs, _ := os.ReadDir(root)
for _, p := range procs {
pid, err := strconv.Atoi(p.Name())
if err != nil {
continue
}
fds, err := os.ReadDir(filepath.Join(root, p.Name(), "fd"))
if err != nil {
continue
}
for _, fd := range fds {
link, err := os.Readlink(filepath.Join(root, p.Name(), "fd", fd.Name()))
if err != nil {
continue
}
idx, ok := want[link]
if !ok {
continue
}
name, _ := os.ReadFile(filepath.Join(root, p.Name(), "comm"))
for _, i := range idx {
if sockets[i].Process == "" {
sockets[i].Process = fmt.Sprintf("%d/%s", pid, strings.TrimSpace(string(name)))
}
}
}
}
unknown := 0
for i := range sockets {
if sockets[i].Process == "" {
sockets[i].Owner = "not readable by this tool's account"
unknown++
}
}
return unknown
}
+33 -3
View File
@@ -1,12 +1,14 @@
// netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches
// and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's
// own sockets and resolver — a TCP connect and a DNS lookup — and nothing that changes anything.
// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named),
// and what listens here — and nothing that changes anything.
// The module's HTTP check is its TypeScript bundle; the runtime serves both under one module.
package main
import (
"fmt"
"os"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -21,6 +23,29 @@ func main() {
func tools() []stdio.Tool {
return []stdio.Tool{
{
Name: "netcheck_listening",
Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " +
"from the kernel's own tables (what ss -lunt reads), with the program holding it where this tool may see it. " +
"port narrows it to one port, e.g. 53 to see on which addresses the resolver answers.",
Input: map[string]any{
"port": map[string]any{"type": "integer", "description": "only this port (default every port)"},
},
Run: func(args map[string]any) (any, error) {
port := 0
if _, given := args["port"]; given {
p, err := whole(args, "port", 0)
if err != nil {
return nil, err
}
if p < 1 || p > 65535 {
return nil, fmt.Errorf("port %d is not a port (1-65535)", p)
}
port = p
}
return Listening("/proc", port), nil
},
},
{
Name: "netcheck_tcp",
Description: "Check whether a TCP port is reachable from this machine: opens one connection " +
@@ -48,11 +73,14 @@ func tools() []stdio.Tool {
},
{
Name: "netcheck_dns",
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts). " +
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " +
"that one resolver alone over UDP (or TCP), answering its response code (NOERROR, NXDOMAIN, REFUSED…). " +
"type is A, AAAA, CNAME, TXT or MX; answers the records found, or the error.",
Input: map[string]any{
"name": map[string]any{"type": "string", "description": "the name to look up"},
"type": map[string]any{"type": "string", "enum": []string{"A", "AAAA", "CNAME", "TXT", "MX"}, "description": "record type (default A)"},
"server": map[string]any{"type": "string", "description": "ask this resolver alone: an address, with :port or without (53), e.g. 10.10.0.1"},
"tcp": map[string]any{"type": "boolean", "description": "ask the server over TCP rather than UDP (default false)"},
"timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 3000, at most 30000)"},
},
Run: func(args map[string]any) (any, error) {
@@ -65,7 +93,9 @@ func tools() []stdio.Tool {
if err != nil {
return nil, err
}
return CheckDNS(name, kind, timeout)
server, _ := args["server"].(string)
overTCP, _ := args["tcp"].(bool)
return AskDNS(name, kind, strings.TrimSpace(server), overTCP, timeout)
},
},
}
+13 -1
View File
@@ -4,8 +4,20 @@
"tools": [
"netcheck_tcp",
"netcheck_dns",
"netcheck_http"
"netcheck_http",
"netcheck_listening"
],
"replaces": {
"netcheck_listening": [
"ss -lunt",
"ss -ltn",
"netstat -lnt"
],
"netcheck_dns": [
"dig",
"nslookup"
]
},
"build": {
"artifacts": [
{