Merge pull request 'dnsmasq answers or is restarted, binds once; tools for DNS, sockets and the runtime's memory (hq issue 348)' (#161) from fix/the-resolver-answers-or-is-restarted into main

This commit was merged in pull request #161.
This commit is contained in:
2026-10-10 15:48:12 +00:00
30 changed files with 2682 additions and 172 deletions
+2
View File
@@ -7,9 +7,11 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
modules/messenger/cmd/messenger/messenger
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
modules/claude-code/cmd/claude-code/claude-code
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
modules/messenger/messenger
modules/claude-code/claude-code
modules/mesh-watcher/mesh-watcher
modules/dnsmasq/dnsmasq
modules/mesh-delivery/mesh-delivery
+1 -1
View File
@@ -1,4 +1,4 @@
70
69
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
-62
View File
@@ -1,62 +0,0 @@
// dnsmasq's own code, living in the module (novox/hq ADR 0039). dnsmasq here answers the mesh's
// generated wildcard names (<service>.<node>.<suffix>) itself and forwards everything else to fixed
// upstreams — the predecessor's arrangement, so the machine and its containers ask it for the world.
// Its code reads what it was told to answer, and can resolve through itself to prove that it does.
import { readFile } from "node:fs/promises";
import { Resolver } from "node:dns/promises";
export interface AnsweredName {
/** A machine's internal name, e.g. "anchor.internal" — it and everything under it resolve here. */
name: string;
address: string;
}
export class DnsmasqClient {
constructor(
private readonly resolverPath: string,
private readonly address: string,
) {}
/**
* Build from the environment. Both values are node-local facts with mesh-chosen defaults — the
* wildcard file the module is sent, and the loopback address its config listens on (127.0.0.1,
* where the predecessor's resolver answered and every machine's resolv.conf already points) — so
* there is nothing to be unconfigured about; it never throws.
*/
static fromEnv(env: NodeJS.ProcessEnv = process.env): DnsmasqClient {
return new DnsmasqClient(
env.MESH_DNSMASQ_RESOLVER_PATH ?? "/etc/mesh-resolver/nodes.conf",
env.MESH_DNSMASQ_ADDRESS ?? "127.0.0.1",
);
}
/** The names this resolver answers, read from the mesh-generated wildcard file. */
async answeredNames(): Promise<AnsweredName[]> {
let text: string;
try {
text = await readFile(this.resolverPath, "utf8");
} catch {
return []; // not yet on the network, or the file has not been written — no names, not an error
}
const names: AnsweredName[] = [];
for (const line of text.split("\n")) {
// dnsmasq wildcard syntax the mesh writes: address=/<node>.<suffix>/<address>. The file also
// carries local=/<suffix>/, which names no machine and is not matched here.
const match = line.match(/^address=\/([^/]+)\/(.+)$/);
if (match) names.push({ name: match[1], address: match[2] });
}
return names;
}
/** Resolve a name through this node's own resolver — the check that wildcard-resolution answers. */
async resolve(name: string): Promise<string[]> {
const resolver = new Resolver();
resolver.setServers([this.address]);
try {
return await resolver.resolve4(name);
} catch {
return await resolver.resolve6(name);
}
}
}
+200
View File
@@ -0,0 +1,200 @@
package main
import (
"context"
"encoding/binary"
"errors"
"fmt"
"math/rand/v2"
"net"
"strings"
"syscall"
"time"
)
// A real question to the resolver, as any program on the machine asks it: one name, one record type, over
// UDP or TCP, to one address. Written out here rather than taken from a library: the module needs the
// header's answer code and the A records, nothing else, and Go's own resolver would read /etc/hosts and
// /etc/resolv.conf instead of asking the one address it is told (novox/hq issue 348).
// Answer is what one question got.
type Answer struct {
// Server is where it was asked, and Over the transport.
Server string `json:"server"`
Over string `json:"over"`
Name string `json:"name"`
// Answered says a reply came; Code is its answer code (NOERROR, NXDOMAIN, …) and Addresses its A
// records.
Answered bool `json:"answered"`
Code string `json:"code,omitempty"`
Addresses []string `json:"addresses,omitempty"`
// Failed is why no reply came, in words: refused (nothing listens there), no answer within, or what
// the system said.
Failed string `json:"failed,omitempty"`
// Took is how long it took, in milliseconds.
Took int64 `json:"took_ms"`
}
// rcodes are the answer codes by number.
var rcodes = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
// query is one question for name's A record, with id.
func query(name string, id uint16) ([]byte, error) {
name = strings.TrimSuffix(name, ".")
if name == "" {
return nil, errors.New("no name to ask")
}
msg := make([]byte, 12, 64)
binary.BigEndian.PutUint16(msg[0:], id)
binary.BigEndian.PutUint16(msg[2:], 0x0100) // a standard query, recursion desired
binary.BigEndian.PutUint16(msg[4:], 1) // one question
for _, label := range strings.Split(name, ".") {
if label == "" || len(label) > 63 {
return nil, fmt.Errorf("%q is not a name that can be asked", name)
}
msg = append(msg, byte(len(label)))
msg = append(msg, label...)
}
msg = append(msg, 0, 0, 1, 0, 1) // root; type A; class IN
return msg, nil
}
// readAnswer reads a reply to the question with id: its code and A records.
func readAnswer(msg []byte, id uint16) (string, []string, error) {
if len(msg) < 12 {
return "", nil, errors.New("a reply too short to be one")
}
if binary.BigEndian.Uint16(msg[0:]) != id {
return "", nil, errors.New("a reply to another question")
}
flags := binary.BigEndian.Uint16(msg[2:])
code, ok := rcodes[int(flags&0xF)]
if !ok {
code = fmt.Sprintf("RCODE%d", flags&0xF)
}
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
at := 12
for i := 0; i < qd; i++ {
var err error
if at, err = skipName(msg, at); err != nil {
return code, nil, err
}
at += 4
}
var addresses []string
for i := 0; i < an; i++ {
var err error
if at, err = skipName(msg, at); err != nil {
return code, addresses, err
}
if at+10 > len(msg) {
return code, addresses, errors.New("a reply cut short")
}
kind, length := binary.BigEndian.Uint16(msg[at:]), int(binary.BigEndian.Uint16(msg[at+8:]))
at += 10
if at+length > len(msg) {
return code, addresses, errors.New("a reply cut short")
}
if kind == 1 && length == 4 {
addresses = append(addresses, net.IP(msg[at:at+4]).String())
}
at += length
}
return code, addresses, nil
}
// skipName steps over a name in a message, a pointer ending it.
func skipName(msg []byte, at int) (int, error) {
for at < len(msg) {
n := int(msg[at])
switch {
case n == 0:
return at + 1, nil
case n&0xC0 == 0xC0:
return at + 2, nil
}
at += n + 1
}
return at, errors.New("a name runs past the reply")
}
// Ask asks server (an address, port 53 unless given) for name's A record over "udp" or "tcp", within
// timeout.
func Ask(ctx context.Context, server, over, name string, timeout time.Duration) Answer {
if _, _, err := net.SplitHostPort(server); err != nil {
server = net.JoinHostPort(server, "53")
}
a := Answer{Server: server, Over: over, Name: name}
started := time.Now()
defer func() { a.Took = time.Since(started).Milliseconds() }()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
id := uint16(rand.UintN(1 << 16))
q, err := query(name, id)
if err != nil {
a.Failed = err.Error()
return a
}
conn, err := (&net.Dialer{}).DialContext(ctx, over, server)
if err != nil {
a.Failed = failedWords(err, timeout)
a.Took = time.Since(started).Milliseconds()
return a
}
defer conn.Close()
if deadline, ok := ctx.Deadline(); ok {
_ = conn.SetDeadline(deadline)
}
var reply []byte
if over == "tcp" {
framed := binary.BigEndian.AppendUint16(nil, uint16(len(q)))
if _, err = conn.Write(append(framed, q...)); err == nil {
head := make([]byte, 2)
if _, err = readFull(conn, head); err == nil {
reply = make([]byte, binary.BigEndian.Uint16(head))
_, err = readFull(conn, reply)
}
}
} else if _, err = conn.Write(q); err == nil {
buf := make([]byte, 1500)
var n int
n, err = conn.Read(buf)
reply = buf[:n]
}
if err != nil {
a.Failed = failedWords(err, timeout)
a.Took = time.Since(started).Milliseconds()
return a
}
a.Answered = true
a.Code, a.Addresses, err = readAnswer(reply, id)
if err != nil {
a.Failed = err.Error()
}
a.Took = time.Since(started).Milliseconds()
return a
}
func readFull(conn net.Conn, buf []byte) (int, error) {
read := 0
for read < len(buf) {
n, err := conn.Read(buf[read:])
read += n
if err != nil {
return read, err
}
}
return read, nil
}
// failedWords is why no reply came: refused when nothing listens, no answer when nothing came in time.
func failedWords(err error, timeout time.Duration) string {
var timedOut interface{ Timeout() bool }
switch {
case errors.Is(err, syscall.ECONNREFUSED):
return "refused: nothing listens there"
case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timedOut) && timedOut.Timeout()):
return fmt.Sprintf("no answer within %s", timeout)
}
return err.Error()
}
@@ -0,0 +1,559 @@
package main
import (
"context"
"encoding/binary"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
// stopped — as dnsmasq answers the machine's own name from the roster.
type standIn struct {
udp net.PacketConn
tcp net.Listener
wg sync.WaitGroup
}
func reply(q []byte, address string) []byte {
// The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name.
out := append([]byte{}, q...)
binary.BigEndian.PutUint16(out[2:], 0x8180)
binary.BigEndian.PutUint16(out[6:], 1)
ip := net.ParseIP(address).To4()
out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4)
return append(out, ip...)
}
func startStandIn(t *testing.T, at, port, address string) *standIn {
t.Helper()
s := &standIn{}
var err error
if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil {
t.Fatal(err)
}
if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil {
t.Fatal(err)
}
s.wg.Add(2)
go func() {
defer s.wg.Done()
buf := make([]byte, 1500)
for {
n, from, err := s.udp.ReadFrom(buf)
if err != nil {
return
}
_, _ = s.udp.WriteTo(reply(buf[:n], address), from)
}
}()
go func() {
defer s.wg.Done()
for {
c, err := s.tcp.Accept()
if err != nil {
return
}
head := make([]byte, 2)
if _, err := readFull(c, head); err == nil {
q := make([]byte, binary.BigEndian.Uint16(head))
if _, err := readFull(c, q); err == nil {
r := reply(q, address)
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
}
}
_ = c.Close()
}
}()
t.Cleanup(s.stop)
return s
}
func (s *standIn) stop() {
_ = s.udp.Close()
_ = s.tcp.Close()
s.wg.Wait()
}
// freePort is a port free on both loopback addresses the tests use.
func freePort(t *testing.T) string {
t.Helper()
for i := 0; i < 20; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
_ = l.Close()
ok := true
for _, at := range []string{"127.0.0.1", "127.0.0.2"} {
for _, over := range []string{"tcp", "udp"} {
if over == "tcp" {
if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil {
_ = c.Close()
} else {
ok = false
}
} else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil {
_ = c.Close()
} else {
ok = false
}
}
}
if ok {
return port
}
}
t.Fatal("no port free on both loopback addresses")
return ""
}
// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address)
// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2.
func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine {
t.Helper()
root := t.TempDir()
write := func(path, text string) {
p := filepath.Join(root, path)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n")
write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+
"address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n")
if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil {
t.Fatal(err)
}
return &Machine{Root: root, Ask: Ask, Now: now, Port: port}
}
func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) {
listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n")
if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" {
t.Fatalf("listen addresses %v", listen)
}
names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" +
"host-record=novox.internal,10.77.0.1\n")
q, ok := Question(listen, names)
if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" {
t.Fatalf("asks %+v", q)
}
if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" {
t.Fatalf("with no own name in the roster, asks %+v", q)
}
}
// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback,
// and refused every question on its private address. Its unit said healthy. The look does not.
func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) {
port := freePort(t)
m := aResolverMachine(t, port, time.Now)
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" {
t.Fatalf("a resolver answering everywhere: %+v", l)
}
private.stop()
l := m.Look(t.Context())
if l.Healthy || len(l.Wrong) != 2 {
t.Fatalf("a resolver that stopped answering on its private address: %+v", l)
}
if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") ||
!strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") {
t.Fatalf("said as %q", why)
}
}
func TestAWrongAnswerIsNotAnAnswer(t *testing.T) {
q := Named{Name: "novox.internal", Address: "10.77.0.1"}
for _, c := range []struct {
a Answer
want string
}{
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""},
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"},
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"},
{Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"},
} {
if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
t.Errorf("%+v said %q, want %q", c.a, got, c.want)
}
}
}
func failing(at time.Time) Look {
return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}},
Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}}
}
func healthy(at time.Time) Look { return Look{At: at, Healthy: true} }
// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks
// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why.
func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
var s State
var why string
if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 {
t.Fatalf("one failing look restarted: %q", why)
}
if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
t.Fatalf("a second failing look did not restart: %+v %q", s, why)
}
s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
// It helped.
s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active")
if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
t.Fatalf("a healthy look after a restart: %+v %q", s, why)
}
// Failing again soon after: held for the gap, then restarted; and so on until the budget is spent.
at := t0.Add(3 * LookEvery)
restarts := 1
for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
at = at.Add(LookEvery)
s, why = Step(s, failing(at), "active")
if why == "" {
continue
}
last := s.Restarts[len(s.Restarts)-1].At
if at.Sub(last) < RestartGap {
t.Fatalf("restarted %s after the last", at.Sub(last))
}
s.Restarts = append(s.Restarts, Restart{At: at, Why: why})
restarts++
}
if restarts != RestartsPerHour {
t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour)
}
if !strings.Contains(s.Held, "the most the watch makes") {
t.Fatalf("the spent budget is not said: %+v", s)
}
for _, r := range s.Restarts[1:] {
if r.Helped == nil || *r.Helped {
t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped)
}
}
// A look that could not ask — no configuration — restarts nothing.
var none State
broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
none, _ = Step(none, broken, "active")
if _, why := Step(none, broken, "active"); why != "" {
t.Fatalf("a missing configuration restarted the resolver: %q", why)
}
}
// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the
// watch notices on its second look, restarts the unit (which brings the listener back), says the restart
// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not
// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the
// operator.
func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) {
port := freePort(t)
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
var mu sync.Mutex
now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() }
m := aResolverMachine(t, port, now)
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
private.stop()
fixes := true
var asked []string
run := func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
if fixes {
private = startStandIn(t, "127.0.0.2", port, "127.0.0.2")
}
return "", nil
}
var s State
look := func() {
t.Helper()
next, why := Step(s, m.Look(t.Context()), "active")
if why != "" {
r := Restart{At: now().UTC(), Why: why}
if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
r.Error = err.Error()
}
next.Restarts = append(next.Restarts, r)
}
s = next
if err := m.WriteState(s); err != nil {
t.Fatal(err)
}
pass(LookEvery)
}
look()
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") {
t.Fatalf("after one failing look: %+v", h.Why)
}
look()
if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" {
t.Fatalf("restarted with %v", asked)
}
look()
if r := s.Restarts[0]; r.Helped == nil || !*r.Helped {
t.Fatalf("the restart that brought it back is said as %+v", r)
}
if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 {
t.Fatalf("after the restart helped: %+v", h)
}
// It stops again, and this time a restart does not help.
private.stop()
fixes = false
pass(RestartGap)
for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ {
look()
}
h := m.Health(t.Context())
if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") ||
!strings.Contains(h.Why, "which did not help") {
t.Fatalf("a restart that did not help: %q", h.Why)
}
// And a watch that stopped looking is said, whatever the resolver does.
pass(Stale + time.Minute)
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") {
t.Fatalf("a stale watch: %q", h.Why)
}
}
// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came
// back on its own, or by something else.
func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
s := State{Restarts: []Restart{{At: t0}}}
s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active")
if h := s.Restarts[0].Helped; h == nil || *h {
t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h)
}
s = State{Restarts: []Restart{{At: t0}}}
s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active")
if h := s.Restarts[0].Helped; h == nil || !*h {
t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h)
}
}
// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is
// held, said, and not counted against the budget (review of PR 161, C2).
func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
for _, unit := range []string{"inactive", "activating", "failed", ""} {
var s State
var why string
for i := 0; i < 4; i++ {
s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit)
if why != "" {
t.Fatalf("a unit %q was restarted", unit)
}
}
if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 {
t.Fatalf("a unit %q: %+v", unit, s)
}
// Running again and still not answering: restarted, the held looks counted as failing.
if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" {
t.Fatalf("a unit running again after %q, not answering, was not restarted", unit)
}
}
}
// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts
// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4).
func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) {
port := freePort(t)
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return clock })
m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer {
return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"}
}
before := LookEvery
LookEvery = time.Millisecond
t.Cleanup(func() { LookEvery = before })
run := func(spent []Restart) ([]string, State) {
t.Helper()
if err := m.WriteState(State{Restarts: spent}); err != nil {
t.Fatal(err)
}
var asked []string
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
looks := 0
var last State
err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
if args[0] == "show" {
return "active\n", nil
}
return "", nil
}, func(s State) error {
last = s
if looks++; looks == 3 {
cancel()
}
return m.WriteState(s)
})
if err != nil {
t.Fatal(err)
}
return asked, last
}
// Three restarts within the hour before it was restarted: it restarts nothing more.
spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}}
asked, last := run(spent)
for _, a := range asked {
if strings.Contains(a, "restart") {
t.Fatalf("restarted with its budget spent before it started: %v", asked)
}
}
if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 {
t.Fatalf("the budget it kept: %+v", last)
}
// None spent: restarted on its second failing look, not waiting for the start.
asked, last = run(nil)
restarts := 0
for _, a := range asked {
if strings.HasPrefix(a, "systemctl restart") {
restarts++
if a != "systemctl restart --no-block dnsmasq.service" {
t.Fatalf("restarted as %q", a)
}
}
}
if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" {
t.Fatalf("restarts %d: %v %+v", restarts, asked, last)
}
}
// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running.
func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) {
port := freePort(t)
clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return clock })
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") {
t.Fatalf("no watch: %+v", h.Why)
}
if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") {
t.Fatalf("a stale watch: %+v", h.Why)
}
if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); !h.Healthy || h.Why != "" {
t.Fatalf("a watch looking, a resolver answering: %+v", h)
}
if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") {
t.Fatalf("a unit stopped: %+v", h.Why)
}
if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") {
t.Fatalf("an unreadable state: %+v", h.Why)
}
}
// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3).
func TestResolveAsksOnlyAListenAddress(t *testing.T) {
port := freePort(t)
m := aResolverMachine(t, port, time.Now)
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") {
t.Fatalf("another address was asked: %v", err)
}
got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp")
if err != nil {
t.Fatal(err)
}
answers := got.(map[string]any)["answers"].([]Answer)
if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" {
t.Fatalf("%+v", answers)
}
if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 {
t.Fatalf("every listen address over both: %+v", got)
}
if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil {
t.Fatal("over quic was accepted")
}
if _, err := m.Resolve(t.Context(), "", "", ""); err == nil {
t.Fatal("no name was accepted")
}
}
// The announcer: names primed silently, restarts made before the bundle started said, an event remembered
// only once emitted, and whether a restart helped said (review of PR 161, C5).
func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) {
port := freePort(t)
t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return t0 })
yes := true
if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil {
t.Fatal(err)
}
var said []string
failing := true
emit := func(kind string, body any, _ ...stdio.EmitOptions) error {
if failing && kind == "restarted" {
return errors.New("no bus")
}
said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"]))
return nil
}
var a Announcer
Tell(m, &a, emit)
if strings.Join(said, "|") != "restart-judged <nil> true" {
t.Fatalf("at the start: %v", said)
}
failing, said = false, nil
Tell(m, &a, emit)
if strings.Join(said, "|") != "restarted <nil> <nil>" {
t.Fatalf("a restart whose emit failed was not said again: %v", said)
}
said = nil
Tell(m, &a, emit)
if len(said) != 0 {
t.Fatalf("said twice: %v", said)
}
// The roster changes: one machine added, one gone. An unreadable roster says nothing.
nodes := filepath.Join(m.Root, NodesFile)
if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil {
t.Fatal(err)
}
Tell(m, &a, emit)
if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") ||
!strings.Contains(got, "name.removed ace.internal") || len(said) != 2 {
t.Fatalf("the roster's change: %v", said)
}
said = nil
if err := os.Remove(nodes); err != nil {
t.Fatal(err)
}
Tell(m, &a, emit)
if len(said) != 0 {
t.Fatalf("an unreadable roster said %v", said)
}
}
+207
View File
@@ -0,0 +1,207 @@
package main
import (
"bufio"
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
// A look asks the resolver one real question on every address its configuration tells it to answer on,
// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat
// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its
// journal silent, and refused every question on its private address for twenty minutes; its unit said
// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers.
// The files a look reads, under the machine's root.
const (
ConfigFile = "/etc/dnsmasq.conf"
NodesFile = "/etc/mesh-resolver/nodes.conf"
)
// AskWithin bounds each question.
const AskWithin = 2 * time.Second
// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock.
type Machine struct {
Root string
Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer
Now func() time.Time
// Port is where the resolver answers; 53 but in a test.
Port string
}
// Here is this machine.
func Here() *Machine {
return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"}
}
func (m *Machine) read(path string) (string, error) {
b, err := os.ReadFile(filepath.Join(m.Root, path))
return string(b), err
}
// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order.
func ListenAddresses(config string) []string {
var out []string
sc := bufio.NewScanner(strings.NewReader(config))
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
value, ok := strings.CutPrefix(line, "listen-address=")
if !ok {
continue
}
for _, a := range strings.Split(value, ",") {
if a = strings.TrimSpace(a); a != "" {
out = append(out, a)
}
}
}
return out
}
// Named is a machine's name and its private address, as the roster gave them to the resolver.
type Named struct {
Name string `json:"name"`
Address string `json:"address"`
}
// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address:
// its `host-record=<name>,<address>` lines.
func AnsweredNames(nodes string) []Named {
var out []Named
sc := bufio.NewScanner(strings.NewReader(nodes))
for sc.Scan() {
value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=")
if !ok {
continue
}
name, address, ok := strings.Cut(value, ",")
if ok && name != "" && address != "" {
out = append(out, Named{Name: name, Address: strings.TrimSpace(address)})
}
}
return out
}
// Question is the name a look asks, and the address a right answer carries: the machine's own when the
// roster names an address the resolver listens on, else the roster's first.
func Question(listen []string, names []Named) (Named, bool) {
for _, n := range names {
for _, a := range listen {
if n.Address == a {
return n, true
}
}
}
if len(names) > 0 {
return names[0], true
}
return Named{}, false
}
// Look is one look: every listen address asked over both transports.
type Look struct {
At time.Time `json:"at"`
// Healthy says every question was answered rightly.
Healthy bool `json:"healthy"`
Asked Named `json:"asked"`
Answers []Answer `json:"answers"`
// Wrong is every question not answered rightly, in words; Problems what kept the look from asking.
Wrong []string `json:"wrong,omitempty"`
Problems []string `json:"problems,omitempty"`
}
// Why is what is wrong in one sentence: empty when healthy.
func (l Look) Why() string {
if l.Healthy {
return ""
}
return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ")
}
// Look asks every listen address its question, over UDP and TCP, all at once.
func (m *Machine) Look(ctx context.Context) Look {
l := Look{At: m.Now().UTC()}
config, err := m.read(ConfigFile)
if err != nil {
l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err))
return l
}
listen := ListenAddresses(config)
if len(listen) == 0 {
l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile))
return l
}
nodes, err := m.read(NodesFile)
if err != nil {
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err))
return l
}
q, ok := Question(listen, AnsweredNames(nodes))
if !ok {
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile))
return l
}
l.Asked = q
var mu sync.Mutex
var wg sync.WaitGroup
for _, address := range listen {
for _, over := range []string{"udp", "tcp"} {
wg.Add(1)
go func(address, over string) {
defer wg.Done()
a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin)
mu.Lock()
l.Answers = append(l.Answers, a)
mu.Unlock()
}(address, over)
}
}
wg.Wait()
sort.Slice(l.Answers, func(i, j int) bool {
if l.Answers[i].Server != l.Answers[j].Server {
return l.Answers[i].Server < l.Answers[j].Server
}
return l.Answers[i].Over > l.Answers[j].Over
})
for _, a := range l.Answers {
if w := wrongWords(a, q); w != "" {
l.Wrong = append(l.Wrong, w)
}
}
l.Healthy = len(l.Wrong) == 0
return l
}
// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the
// machine's address.
func wrongWords(a Answer, q Named) string {
at := a.Server + " over " + a.Over
switch {
case !a.Answered:
return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed)
case a.Code != "NOERROR":
return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code)
case a.Failed != "":
return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed)
}
for _, x := range a.Addresses {
if x == q.Address {
return ""
}
}
return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address)
}
func orNothing(s []string) string {
if len(s) == 0 {
return "no address"
}
return strings.Join(s, ", ")
}
+145
View File
@@ -0,0 +1,145 @@
// The dnsmasq module's Go bundle: one of the mesh's resolvers (novox/hq ADR 0194, 0223).
//
// Started with no arguments it is served by the node's runtime as the operator account, over MCP on stdio
// (ADR 0188, ADR 0193): the module's tools and its health, and beside them the events the resolver's
// answers give — a machine's name it now answers or no longer does, and a restart the watch made. Started
// as `dnsmasq-tools watch` it is the module's long-running process, as root: it asks the resolver a real
// question on every listen address and restarts it, within a budget, when it stops answering (watch.go,
// novox/hq issue 348). stdout is the MCP channel; everything else is said on stderr.
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func main() {
if len(os.Args) > 1 {
if os.Args[1] != "watch" || len(os.Args) != 2 {
fmt.Fprintln(os.Stderr, "usage: dnsmasq-tools [watch]")
os.Exit(2)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
defer stop()
m := Here()
if err := m.Watch(ctx, ExecRunner, m.WriteState); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
return
}
m := Here()
go Announce(m, stdio.Emit)
if err := stdio.Serve("", Tools(m)); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
// AnnounceEvery is how often the bundle reads the roster and the watch for what to say.
var AnnounceEvery = 30 * time.Second
// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not
// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when
// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded.
type Announcer struct {
names map[string]string
primed bool
// said is every restart said, by when it was made; judged every one whose outcome was said.
said, judged map[time.Time]bool
}
// Event is one event to emit, and what saying it records.
type Event struct {
Type string
Body map[string]any
said func()
}
// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does
// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events;
// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it
// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names,
// rather than every name removed. An event is remembered by Said, once it was emitted.
func (a *Announcer) Next(names []Named, read bool, s State) []Event {
if a.said == nil {
a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{}
}
var out []Event
if read {
now := map[string]string{}
for _, n := range names {
now[n.Name] = n.Address
}
if !a.primed {
a.names, a.primed = now, true
} else {
for name, address := range now {
if _, had := a.names[name]; !had {
out = append(out, Event{"name.added", map[string]any{"name": name, "address": address},
func() { a.names[name] = address }})
}
}
for name := range a.names {
if _, has := now[name]; !has {
out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }})
}
}
}
}
for _, r := range s.Restarts {
at := r.At
if !a.said[at] {
body := map[string]any{"unit": Unit, "at": at, "why": r.Why}
if r.Error != "" {
body["error"] = r.Error
}
out = append(out, Event{"restarted", body, func() { a.said[at] = true }})
}
if r.Helped != nil && !a.judged[at] {
out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped},
func() { a.judged[at] = true }})
}
}
return out
}
// Said remembers an event as said.
func (e Event) Said() {
if e.said != nil {
e.said()
}
}
// Announce says the events for as long as the bundle runs.
func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) {
time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs
var a Announcer
for {
Tell(m, &a, emit)
time.Sleep(AnnounceEvery)
}
}
// Tell is one read and what it says: an event that could not be emitted is said again at the next read.
func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) {
var names []Named
text, err := m.read(NodesFile)
if err == nil {
names = AnsweredNames(text)
}
s, _, _ := m.ReadState()
for _, e := range a.Next(names, err == nil, s) {
if err := emit(e.Type, e.Body); err != nil {
fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err)
continue
}
e.Said()
}
}
+166
View File
@@ -0,0 +1,166 @@
package main
import (
"context"
"fmt"
"slices"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// Stale is how old the watch's last look may be before the health tool says it stopped looking.
const Stale = 2 * time.Minute
// Health is the module's health answer (novox/hq ADR 0240): healthy only when the resolver answers its
// real question on every listen address, over UDP and TCP, and the watch is looking.
type Health struct {
Healthy bool `json:"healthy"`
Why string `json:"why,omitempty"`
Look Look `json:"look"`
// Watch is what the watch last saw and did; absent when it has written nothing yet.
Watch *State `json:"watch,omitempty"`
}
// Health asks the resolver now, and reads what the watch did.
func (m *Machine) Health(ctx context.Context) Health {
l := m.Look(ctx)
h := Health{Healthy: l.Healthy, Look: l}
var why []string
if !l.Healthy {
why = append(why, "the resolver does not answer as it must: "+l.Why())
}
s, had, err := m.ReadState()
switch {
case err != nil:
h.Healthy = false
why = append(why, "what the watch saw cannot be read: "+err.Error())
case !had:
h.Healthy = false
why = append(why, "the watch has written nothing yet, so nothing restarts the resolver when it stops answering")
default:
h.Watch = &s
if age := m.Now().Sub(s.Last.At); age > Stale {
h.Healthy = false
why = append(why, fmt.Sprintf("the watch last looked %s ago, so nothing restarts the resolver when it "+
"stops answering", age.Round(time.Second)))
}
if !l.Healthy {
why = append(why, restartWords(s))
}
if s.Unit != "" && s.Unit != "active" {
why = append(why, "the unit is "+s.Unit)
}
}
h.Why = strings.Join(nonEmpty(why), "; ")
return h
}
// restartWords is what the watch did about it, in words.
func restartWords(s State) string {
if len(s.Restarts) == 0 {
return "the watch has not restarted it" + orEmpty("; ", s.Held)
}
r := s.Restarts[len(s.Restarts)-1]
words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC"))
switch {
case r.Error != "":
words += ", and the restart failed: " + r.Error
case r.Helped != nil && !*r.Helped:
words += ", which did not help"
case r.Helped == nil:
words += ", and its next looks decide whether that helped"
}
if s.Held != "" {
words += "; " + s.Held
}
return words
}
func nonEmpty(s []string) []string {
var out []string
for _, x := range s {
if x != "" {
out = append(out, x)
}
}
return out
}
func str(description string) map[string]any {
return map[string]any{"type": "string", "description": description}
}
func arg(a map[string]any, k string) string {
v, _ := a[k].(string)
return strings.TrimSpace(v)
}
// Tools are the module's tools.
func Tools(m *Machine) []stdio.Tool {
return []stdio.Tool{
{Name: "dnsmasq_health", Description: "Whether this machine's mesh resolver answers: its real question — this " +
"machine's own name — asked now on every address its configuration listens on, over UDP and TCP, each " +
"answer with its code, addresses and time; and what the module's watch last saw, the restarts it made " +
"within the hour, whether each helped, and why it holds back from another. Healthy only when every " +
"question was answered rightly and the watch is looking (novox/hq ADR 0240, issue 348). (r)",
Input: map[string]any{},
Run: func(map[string]any) (any, error) {
return m.Health(context.Background()), nil
}},
{Name: "dnsmasq_names", Description: "The mesh's machine names this machine's resolver answers, each with its " +
"private address, as the roster wrote them (" + NodesFile + "). (r)",
Input: map[string]any{},
Run: func(map[string]any) (any, error) {
text, err := m.read(NodesFile)
if err != nil {
return nil, fmt.Errorf("the roster %s cannot be read: %w", NodesFile, err)
}
return map[string]any{"names": AnsweredNames(text)}, nil
}},
{Name: "dnsmasq_resolve", Description: "Ask this machine's resolver one name's IPv4 address, directly, on one " +
"address it listens on (default every one) over udp or tcp (default both): the answer code, the addresses " +
"and the time each took, or why none came — refused when nothing listens there. Replaces dig @address. (r)",
Input: map[string]any{"name": str("the name to ask, e.g. plex.anchor.internal"),
"address": str("one listen address to ask (default: every one in its configuration)"),
"over": str("udp or tcp (default: both)")},
Run: func(a map[string]any) (any, error) {
return m.Resolve(context.Background(), arg(a, "name"), arg(a, "address"), arg(a, "over"))
}},
}
}
// Resolve asks name on one listen address or every one, over one transport or both.
func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, error) {
if name == "" {
return nil, fmt.Errorf("name is required")
}
config, err := m.read(ConfigFile)
if err != nil {
return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err)
}
addresses := ListenAddresses(config)
if address != "" {
if !slices.Contains(addresses, address) {
return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver",
address, strings.Join(addresses, ", "))
}
addresses = []string{address}
}
overs := []string{"udp", "tcp"}
switch over {
case "":
case "udp", "tcp":
overs = []string{over}
default:
return nil, fmt.Errorf("over is udp or tcp, not %q", over)
}
var answers []Answer
for _, a := range addresses {
for _, o := range overs {
answers = append(answers, m.Ask(ctx, a+":"+m.Port, o, name, AskWithin))
}
}
return map[string]any{"name": name, "answers": answers}, nil
}
+240
View File
@@ -0,0 +1,240 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real
// question on every listen address (look.go); after Confirm failing looks in a row it restarts the
// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh
// (novox/hq issue 348).
//
// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that
// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor
// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where
// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked,
// and it never says it worked: its next look does. A restart that did not help is said by the health
// tool as the resolver's failure, which the node-engine states and the controller raises — the operator
// hears of it only then.
// The watch's clocks and its budget.
var (
LookEvery = 15 * time.Second
// Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277).
Confirm = 2
// RestartGap is the least time between two restarts; RestartsPerHour the most within an hour.
RestartGap = 2 * time.Minute
RestartsPerHour = 3
// HelpedWithin is how soon after a restart a healthy look says the restart helped.
HelpedWithin = time.Minute
)
// Unit is the resolver's service.
const Unit = "dnsmasq.service"
// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator
// account the tools run as, written only by the watch.
const StateFile = "/var/lib/dnsmasq-watch/watch.json"
// Restart is one restart the watch made.
type Restart struct {
At time.Time `json:"at"`
Why string `json:"why"`
// Error is what the service manager said when the restart itself failed.
Error string `json:"error,omitempty"`
// Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once
// HelpedWithin passed without one, absent until then.
Helped *bool `json:"helped,omitempty"`
}
// State is the watch's last look and what it did, as the tools read it.
type State struct {
Last Look `json:"last"`
// Failing is how many looks in a row failed; Since when the first of them looked.
Failing int `json:"failing"`
Since *time.Time `json:"since,omitempty"`
// Restarts are the restarts within the last hour and the last one ever, newest last.
Restarts []Restart `json:"restarts,omitempty"`
// Held is why a restart that was due was not made: the budget is spent, or the unit is not running.
Held string `json:"held,omitempty"`
// Unit is the resolver unit's state at the last look, as the service manager said it: active,
// activating, inactive, failed …
Unit string `json:"unit,omitempty"`
}
// Runner runs one command; injected, so the watch is tested over recorded answers.
type Runner func(ctx context.Context, name string, args ...string) (string, error)
// ExecRunner runs a command on the machine, bounded to a minute.
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
var out bytes.Buffer
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stdout, cmd.Stderr = &out, &out
err := cmd.Run()
return strings.TrimSpace(out.String()), err
}
// Step is one look of the watch and what follows from it: the state after, and whether to restart and why.
// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of
// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at
// boot for the private address — or failed is the service manager's, and none of them is a resolver that
// runs and does not answer. Pure.
func Step(s State, l Look, unit string) (State, string) {
now := l.At
s.Last, s.Unit = l, unit
// What the earlier restarts are now known to have done.
for i := range s.Restarts {
r := &s.Restarts[i]
if r.Helped != nil || r.Error != "" {
continue
}
switch {
case l.Healthy && now.Sub(r.At) <= HelpedWithin:
yes := true
r.Helped = &yes
case now.Sub(r.At) > HelpedWithin:
no := false
r.Helped = &no
}
}
// The budget's window: the last hour, and always the newest.
kept := s.Restarts[:0]
for i, r := range s.Restarts {
if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 {
kept = append(kept, r)
}
}
s.Restarts = kept
if l.Healthy {
s.Failing, s.Since, s.Held = 0, nil, ""
return s, ""
}
if s.Failing == 0 {
at := now
s.Since = &at
}
s.Failing++
if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 {
// Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing
// configuration or roster is missing.
return s, ""
}
if unit != "active" {
s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+
"answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit))
return s, ""
}
within := 0
var newest *Restart
for i, r := range s.Restarts {
if now.Sub(r.At) <= time.Hour {
within++
}
newest = &s.Restarts[i]
}
switch {
case newest != nil && now.Sub(newest.At) < RestartGap:
s.Held = ""
return s, "" // the last restart is still settling
case within >= RestartsPerHour:
s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+
"more until the oldest is an hour old, and the resolver's failure is said as its health", within)
return s, ""
}
s.Held = ""
return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why())
}
// Watch runs the watch until ctx ends.
func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error {
var s State
if raw, err := m.read(StateFile); err == nil {
_ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart
}
tick := time.NewTicker(LookEvery)
defer tick.Stop()
for {
l := m.Look(ctx)
unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit)
if err != nil {
unit = ""
}
next, why := Step(s, l, strings.TrimSpace(unit))
if why != "" {
r := Restart{At: m.Now().UTC(), Why: why}
// Not waited for: a start that waits for the private address is the service manager's to time,
// and the next looks say whether it answers.
if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil {
r.Error = strings.TrimSpace(err.Error() + " " + out)
}
fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error))
next.Restarts = append(next.Restarts, r)
} else if !l.Healthy && next.Failing == Confirm {
fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held))
}
s = next
if err := write(s); err != nil {
fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err)
}
select {
case <-ctx.Done():
return nil
case <-tick.C:
}
}
}
func orUnknown(s string) string {
if s == "" {
return "not known (the service manager did not say)"
}
return s
}
func orEmpty(prefix, s string) string {
if s == "" {
return ""
}
return prefix + s
}
// WriteState writes the state whole, replacing the file in one rename.
func (m *Machine) WriteState(s State) error {
path := filepath.Join(m.Root, StateFile)
body, err := json.MarshalIndent(s, "", " ")
if err != nil {
return err
}
tmp := path + ".new"
if err := os.WriteFile(tmp, body, 0o644); err != nil {
return err
}
return os.Rename(tmp, path)
}
// ReadState is the watch's last state, and false when it has written none.
func (m *Machine) ReadState() (State, bool, error) {
raw, err := m.read(StateFile)
if errors.Is(err, os.ErrNotExist) {
return State{}, false, nil
}
if err != nil {
return State{}, false, err
}
var s State
if err := json.Unmarshal([]byte(raw), &s); err != nil {
return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err)
}
return s, true, nil
}
+5
View File
@@ -0,0 +1,5 @@
module dnsmasq
go 1.25.0
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
-38
View File
@@ -1,38 +0,0 @@
// dnsmasq's events. The resolver's answered set changes whenever the mesh rewrites the wildcard file
// and restarts it — a machine joined the private network or left it. This watches that file and
// announces, from the resolver's own vantage, that a name became (or stopped being) resolvable on
// this node: DNS having actually propagated here, distinct from the mesh's own node.* events.
//
// Emits (novox/hq ADR 0041/0042):
// module.dnsmasq.name.added — this node's resolver now answers a machine's name
// module.dnsmasq.name.removed — it no longer does
import { emit } from "@novox/mesh-sdk/events";
import { DnsmasqClient } from "./client.js";
const dnsmasq = DnsmasqClient.fromEnv();
// name -> address, primed silently so a restart does not re-announce every name it already answered.
const known = new Map<string, string>();
let primed = false;
async function poll(): Promise<void> {
const now = new Map((await dnsmasq.answeredNames()).map((a) => [a.name, a.address]));
if (primed) {
for (const [name, address] of now) {
if (!known.has(name)) await emit("name.added", { name, address });
}
for (const [name] of known) {
if (!now.has(name)) await emit("name.removed", { name });
}
}
known.clear();
for (const [name, address] of now) known.set(name, address);
primed = true;
}
const run = (): void => void poll().catch((err) => console.error(`[dnsmasq] ${err}`));
setInterval(run, 30_000);
run();
console.log("[dnsmasq] watching the resolver's answered names");
File diff suppressed because one or more lines are too long
-9
View File
@@ -1,9 +0,0 @@
{
"name": "@novox/module-dnsmasq",
"version": "0.1.0",
"description": "dnsmasq — the mesh's resolver: answers wildcard node names and forwards the rest upstream. Its tools and events live here.",
"type": "module",
"private": true,
"dependencies": { "@novox/mesh-sdk": "^0.1.0" },
"devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" }
}
-31
View File
@@ -1,31 +0,0 @@
// dnsmasq's tools — a resolver's two useful questions: what does it answer, and does it answer a
// given name. Moved into the module (novox/hq ADR 0039); the mesh serves them through the sdk.
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { DnsmasqClient } from "../client.js";
export function getDnsmasqTools(dnsmasq: DnsmasqClient): ToolDefinition[] {
return [
{
name: "dnsmasq_names",
description: "The mesh names this node's resolver answers — each a machine and everything under it.",
input: {},
run: async () => ({ names: await dnsmasq.answeredNames() }),
},
{
name: "dnsmasq_resolve",
description: "Resolve a mesh name through this node's own resolver — the check that wildcard-resolution answers.",
input: { name: { type: "string", description: "a name to resolve, e.g. plex.anchor.internal" } },
run: async (args) => {
const name = String(args.name);
try {
return { name, addresses: await dnsmasq.resolve(name) };
} catch (err) {
return { name, addresses: [], error: err instanceof Error ? err.message : String(err) };
}
},
},
];
}
registerModuleTools("dnsmasq", (env) => getDnsmasqTools(DnsmasqClient.fromEnv(env)));
-12
View File
@@ -1,12 +0,0 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"noEmit": true
},
"include": ["client.ts", "index.ts", "tools/index.ts"]
}
+63 -4
View File
@@ -991,30 +991,53 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
}
// Events is what the runtime did in a window ending now, the latest last.
//
// **The runtime keeps only its last events in memory** (about 256), and the images' own health checks
// make three exec events each per look: on a machine with a few checked containers that history is ten
// minutes long or less, all of it exec events (hq research 032). A window asked for further back than
// the history reaches is answered with what is still held AND says where the history begins, so an
// empty list never reads as "nothing happened" (hq issue 348: three hours asked across an outage,
// answered zero). The type filter is applied here rather than by the runtime, for the same reason: the
// history's start is only known from all of it.
func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
return c.eventsAt(ctx, time.Now(), minutes, kind, limit, execs)
}
func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
if kind != "" {
switch kind {
case "container", "image", "network", "volume", "daemon", "plugin", "builder":
default:
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
}
args = append(args, "--filter", "type="+kind)
}
// The whole of what the runtime holds, from the epoch: its buffer, which says where its history really
// begins and whether it is full; the window is cut from it here.
args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"}
out, err := c.docker(ctx, args...)
if err != nil {
return nil, err
}
raw, err := jsonLines[runtimeEvent](out)
all, err := jsonLines[runtimeEvent](out)
if err != nil {
return nil, err
}
start := now.Add(-time.Duration(minutes) * time.Minute)
raw := make([]runtimeEvent, 0, len(all))
for _, e := range all {
if !time.Unix(0, e.TimeNano).Before(start) {
raw = append(raw, e)
}
}
// An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer
// is read by agents and kept in their transcripts, which would make it a second copy of the leak.
envs := &envCache{c: c, ctx: ctx}
redacted := map[string]bool{}
events := []map[string]any{}
for _, e := range raw {
if kind != "" && e.Type != kind {
continue
}
if !execs && strings.HasPrefix(e.Action, "exec_") {
continue
}
@@ -1042,14 +1065,50 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int
if len(events) > limit {
events = events[len(events)-limit:]
}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
"held": len(all)}
if len(redacted) > 0 {
answer["redacted"] = sortedSet(redacted)
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
}
for k, v := range historyReach(all, start) {
answer[k] = v
}
return answer, nil
}
// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer
// holding fewer has every event since the runtime started; a full one has lost what came before its oldest.
const RuntimeEventsKept = 256
// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window
// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not
// full — then it holds everything since the runtime started. Otherwise where the history begins, and that
// nothing before it can be answered from here.
func historyReach(all []runtimeEvent, start time.Time) map[string]any {
if len(all) == 0 {
return map[string]any{"complete": true, "oldest_held": nil,
"history": "the runtime holds no events at all: none happened since it started"}
}
oldest := time.Unix(0, all[0].TimeNano)
for _, e := range all {
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
oldest = t
}
}
at := oldest.UTC().Format(time.RFC3339)
if len(all) < RuntimeEventsKept {
return map[string]any{"complete": true, "oldest_held": at,
"history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)}
}
if !oldest.After(start) {
return map[string]any{"complete": true, "oldest_held": at}
}
return map[string]any{"complete": false, "oldest_held": at,
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
}
func sortedSet(set map[string]bool) []string {
out := make([]string, 0, len(set))
for k := range set {
+71 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"reflect"
@@ -299,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
`
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false)
if err != nil {
t.Fatal(err)
}
@@ -307,7 +308,8 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" {
t.Fatalf("%v", evs)
}
if !f.ran("docker events --since 30m --until 0s --format {{json .}} --filter type=container") {
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
@@ -362,3 +364,70 @@ func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) {
t.Fatalf("%v %v", u, err)
}
}
// eventLine is one line of `docker events --format '{{json .}}'`.
func eventLine(at time.Time, kind, action, name string) string {
return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n",
kind, action, name, at.UnixNano())
}
// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the
// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened.
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
var out strings.Builder
for i := 0; i < RuntimeEventsKept; i++ {
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
}
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
if err != nil {
t.Fatal(err)
}
if got["count"] != 0 || got["held"] != RuntimeEventsKept {
t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept)
}
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
t.Fatalf("an empty answer from a short history reads as quiet: %v", got)
}
}
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
// A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer
// is not full, so it holds every event since the runtime started.
out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") +
eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false)
if err != nil {
t.Fatal(err)
}
if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") {
t.Fatalf("a quiet machine's window: %v", got)
}
if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
}
// A quiet machine asked for ten minutes, its newest event an hour old: complete, and none.
got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("a quiet machine read as one whose history was lost: %v", got)
}
// No events at all: none since the runtime started.
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("an empty buffer: %v", got)
}
// A full buffer whose oldest is before the window: complete.
var full strings.Builder
for i := 0; i < RuntimeEventsKept; i++ {
full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x"))
}
got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != true || got["history"] != nil {
t.Fatalf("a full buffer reaching past the window: %v", got)
}
}
+15 -1
View File
@@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool {
return c.Inspect(ctx, ref)
},
},
{
Name: "docker_resolv_conf",
Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " +
"its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.",
Input: map[string]any{"container": containerArg},
Run: func(args map[string]any) (any, error) {
ref, err := text(args, "container")
if err != nil {
return nil, err
}
return c.ResolvConfOf(ctx, ref)
},
},
{
Name: "docker_logs",
Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " +
@@ -262,7 +275,8 @@ func tools(c *Client) []stdio.Tool {
{
Name: "docker_events",
Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " +
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag.",
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag. " +
"The runtime keeps only its last ~256 events in memory, which health-check execs fill in minutes: complete=false and history say where what it holds begins, and an empty list then means gone, not quiet.",
Input: map[string]any{
"minutes": map[string]any{"type": "integer", "description": "how far back (default 60, at most 1440)"},
"type": map[string]any{"type": "string", "description": "only one kind: container, image, network, volume, daemon, plugin or builder"},
+111
View File
@@ -0,0 +1,111 @@
package main
// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's
// resolver file, made by the runtime when it is created and kept under the runtime's root-only
// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the
// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the
// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the
// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream):
// no exec into the container, nothing written, and it works where the program has no shell.
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"strings"
)
// ContainerResolvConf is the path read inside a container.
const ContainerResolvConf = "/etc/resolv.conf"
// HostResolvConf is the machine's own resolver file, read for comparison.
const HostResolvConf = "/etc/resolv.conf"
// ResolvConf is a resolver file, parsed.
type ResolvConf struct {
Nameservers []string `json:"nameservers"`
Search []string `json:"search,omitempty"`
Options []string `json:"options,omitempty"`
Raw string `json:"raw"`
}
// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only.
func parseResolvConf(text string) ResolvConf {
out := ResolvConf{Nameservers: []string{}, Raw: text}
for _, line := range strings.Split(text, "\n") {
f := strings.Fields(line)
if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") {
continue
}
switch f[0] {
case "nameserver":
if len(f) > 1 {
out.Nameservers = append(out.Nameservers, f[1])
}
case "search", "domain":
out.Search = append(out.Search, f[1:]...)
case "options":
out.Options = append(out.Options, f[1:]...)
}
}
return out
}
// fileFromArchive is the one regular file a `docker cp <ref>:<path> -` stream carries.
func fileFromArchive(stream string) (string, error) {
tr := tar.NewReader(strings.NewReader(stream))
for {
h, err := tr.Next()
if errors.Is(err, io.EOF) {
return "", errors.New("the runtime's archive held no file")
}
if err != nil {
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
}
switch h.Typeflag {
case tar.TypeReg:
body, err := io.ReadAll(io.LimitReader(tr, 64<<10))
if err != nil {
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
}
return string(body), nil
case tar.TypeSymlink:
return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname)
}
}
}
// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list
// the same resolvers in the same order.
func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) {
ref, err := Ref(ref)
if err != nil {
return nil, err
}
answer := map[string]any{"container": ref, "path": ContainerResolvConf}
stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-")
if err != nil {
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
}
text, err := fileFromArchive(stream)
if err != nil {
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
}
inside := parseResolvConf(text)
answer["resolv_conf"] = inside
if raw, err := c.ReadFile(HostResolvConf); err != nil {
answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error()
} else {
host := parseResolvConf(string(raw))
answer["machine_nameservers"] = host.Nameservers
same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ")
answer["same_as_machine"] = same
if !same {
answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " +
"or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again"
}
}
return answer, nil
}
@@ -0,0 +1,96 @@
package main
import (
"archive/tar"
"bytes"
"context"
"errors"
"strings"
"testing"
)
func archiveOf(t *testing.T, h tar.Header, body string) string {
t.Helper()
var buf bytes.Buffer
w := tar.NewWriter(&buf)
h.Size = int64(len(body))
if h.Typeflag == tar.TypeSymlink {
h.Size = 0
}
if err := w.WriteHeader(&h); err != nil {
t.Fatal(err)
}
if h.Typeflag != tar.TypeSymlink {
if _, err := w.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
w.Close()
return buf.String()
}
// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy
// of the file could not be read by anything on the mesh.
func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) {
inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n"
f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -",
Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)})
c := client(f, 1000)
c.ReadFile = func(p string) ([]byte, error) {
if p != HostResolvConf {
t.Fatalf("read %s", p)
}
return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil
}
got, err := c.ResolvConfOf(context.Background(), "route-proxy")
if err != nil {
t.Fatal(err)
}
rc := got["resolv_conf"].(ResolvConf)
if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" ||
strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside {
t.Fatalf("parsed: %+v", rc)
}
if got["same_as_machine"] != false || got["note"] == nil {
t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got)
}
if f.ran("docker exec") {
t.Fatal("read by exec, not through the archive")
}
}
func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") {
t.Fatalf("a missing container: %v", err)
}
link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "")
c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") {
t.Fatalf("a link inside: %v", err)
}
c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000)
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil {
t.Fatal("garbage was read as a file")
}
ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n")
c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000)
c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") }
got, err := c.ResolvConfOf(context.Background(), "x")
if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil {
t.Fatalf("an unreadable machine file: %v, %v", got, err)
}
}
// The container is named as docker knows it, never as something docker would read as an option.
func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) {
f := &fake{}
for _, ref := range []string{"--privileged", "a b", "", "x;rm"} {
if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil {
t.Errorf("%q was asked of the runtime", ref)
}
}
if len(f.calls) != 0 {
t.Fatalf("the runtime was asked: %+v", f.calls)
}
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"strings"
"testing"
"time"
)
// The shape of the leak in hq issue 268: a server password announced at start and a database URI
@@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) {
f := (&fake{}).
on("docker events", Ran{Stdout: execEvents}).
on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"})
got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true)
if err != nil {
t.Fatal(err)
}
+6
View File
@@ -18,6 +18,7 @@
"tools": [
"docker_list",
"docker_inspect",
"docker_resolv_conf",
"docker_logs",
"docker_secrets_in_logs",
"docker_secrets_in_events",
@@ -38,6 +39,11 @@
"docker_problems",
"docker_ports"
],
"replaces": {
"docker_resolv_conf": [
"docker exec cat /etc/resolv.conf"
]
},
"resources": [
{
"id": "package",
+252
View File
@@ -0,0 +1,252 @@
package main
// Asking one named resolver directly (hq issue 348). netcheck_dns without a server asks the machine's
// resolver the way its programs do — resolv.conf, hosts, search domains, several servers in turn — which
// answers "can this machine resolve?" and hides which resolver answered. During the 2026-10-09 outage the
// question was "does the resolver at 10.10.0.1 answer?", and nothing on the mesh could ask it. Asked with a
// server, the question goes to that address and port alone, over UDP or TCP, and the answer says its
// response code: a refusal, no such name, and no answer at all are three different findings.
//
// The wire format is written here rather than taken from a library: a question and the records of five
// types are all it needs, and the module stays without a dependency.
import (
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
)
var dnsTypeCodes = map[string]uint16{"A": 1, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28}
var rcodeWords = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
// serverAddress is a server as asked — an address, with a port or without (53) — as host:port.
func serverAddress(server string) (string, error) {
server = strings.TrimSpace(server)
if host, port, err := net.SplitHostPort(server); err == nil {
p, err := strconv.Atoi(port)
if err != nil || p < 1 || p > 65535 {
return "", fmt.Errorf("server %q: the port is not one (1-65535)", server)
}
if net.ParseIP(host) == nil {
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
}
return net.JoinHostPort(host, port), nil
}
if net.ParseIP(server) == nil {
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
}
return net.JoinHostPort(server, "53"), nil
}
// askServer asks one resolver one question and reads its answer.
func askServer(server, name, kind string, overTCP bool, timeout time.Duration) (rcode int, answers []string, err error) {
addr, err := serverAddress(server)
if err != nil {
return 0, nil, err
}
query, id, err := buildQuery(name, dnsTypeCodes[kind])
if err != nil {
return 0, nil, err
}
network := "udp"
if overTCP {
network = "tcp"
}
conn, err := net.DialTimeout(network, addr, timeout)
if err != nil {
return 0, nil, err
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(timeout))
var reply []byte
if overTCP {
framed := binary.BigEndian.AppendUint16(nil, uint16(len(query)))
if _, err := conn.Write(append(framed, query...)); err != nil {
return 0, nil, err
}
var size [2]byte
if _, err := readFull(conn, size[:]); err != nil {
return 0, nil, err
}
reply = make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := readFull(conn, reply); err != nil {
return 0, nil, err
}
} else {
if _, err := conn.Write(query); err != nil {
return 0, nil, err
}
buf := make([]byte, 4096)
n, err := conn.Read(buf)
if err != nil {
return 0, nil, err
}
reply = buf[:n]
}
return parseReply(reply, id, dnsTypeCodes[kind])
}
func readFull(conn net.Conn, b []byte) (int, error) {
got := 0
for got < len(b) {
n, err := conn.Read(b[got:])
got += n
if err != nil {
return got, err
}
}
return got, nil
}
// buildQuery is one question, recursion desired, under a random id.
func buildQuery(name string, qtype uint16) ([]byte, uint16, error) {
var idb [2]byte
_, _ = rand.Read(idb[:])
id := binary.BigEndian.Uint16(idb[:])
msg := binary.BigEndian.AppendUint16(nil, id)
msg = append(msg, 0x01, 0x00) // recursion desired
msg = append(msg, 0, 1, 0, 0, 0, 0, 0, 0) // one question
wire, err := encodeName(name)
if err != nil {
return nil, 0, err
}
msg = append(msg, wire...)
msg = binary.BigEndian.AppendUint16(msg, qtype)
msg = binary.BigEndian.AppendUint16(msg, 1) // IN
return msg, id, nil
}
func encodeName(name string) ([]byte, error) {
name = strings.TrimSuffix(strings.TrimSpace(name), ".")
var out []byte
if name != "" {
for _, label := range strings.Split(name, ".") {
if label == "" || len(label) > 63 {
return nil, fmt.Errorf("name %q has an empty or too long label", name)
}
out = append(out, byte(len(label)))
out = append(out, label...)
}
}
return append(out, 0), nil
}
var errShort = errors.New("the answer is cut short")
// readName reads a possibly compressed name at off, and the offset after it.
func readName(msg []byte, off int) (string, int, error) {
var labels []string
next, jumped := -1, 0
for {
if off >= len(msg) {
return "", 0, errShort
}
l := int(msg[off])
switch {
case l == 0:
if next < 0 {
next = off + 1
}
return strings.Join(labels, "."), next, nil
case l&0xC0 == 0xC0:
if off+1 >= len(msg) {
return "", 0, errShort
}
if next < 0 {
next = off + 2
}
if jumped++; jumped > 32 {
return "", 0, errors.New("the answer's names point in a loop")
}
off = int(binary.BigEndian.Uint16(msg[off:]) & 0x3FFF)
default:
if off+1+l > len(msg) {
return "", 0, errShort
}
labels = append(labels, string(msg[off+1:off+1+l]))
off += 1 + l
}
}
}
// parseReply reads the response code and the answers of the asked type.
func parseReply(msg []byte, id, qtype uint16) (int, []string, error) {
if len(msg) < 12 {
return 0, nil, errShort
}
if binary.BigEndian.Uint16(msg) != id {
return 0, nil, errors.New("the answer is to another question (its id differs)")
}
if msg[2]&0x80 == 0 {
return 0, nil, errors.New("the answer is not marked as one")
}
rcode := int(msg[3] & 0x0F)
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
off := 12
for i := 0; i < qd; i++ {
_, n, err := readName(msg, off)
if err != nil {
return rcode, nil, err
}
off = n + 4
}
answers := []string{}
for i := 0; i < an; i++ {
_, n, err := readName(msg, off)
if err != nil {
return rcode, answers, err
}
off = n
if off+10 > len(msg) {
return rcode, answers, errShort
}
rtype := binary.BigEndian.Uint16(msg[off:])
size := int(binary.BigEndian.Uint16(msg[off+8:]))
data := off + 10
if data+size > len(msg) {
return rcode, answers, errShort
}
rdata := msg[data : data+size]
off = data + size
if rtype != qtype {
continue // a CNAME on the way to an address, say
}
switch rtype {
case 1, 28:
answers = append(answers, net.IP(rdata).String())
case 5:
s, _, err := readName(msg, data)
if err != nil {
return rcode, answers, err
}
answers = append(answers, s)
case 15:
if size < 3 {
return rcode, answers, errShort
}
s, _, err := readName(msg, data+2)
if err != nil {
return rcode, answers, err
}
answers = append(answers, fmt.Sprintf("%d %s", binary.BigEndian.Uint16(rdata), s))
case 16:
var parts []string
for p := 0; p < len(rdata); {
l := int(rdata[p])
if p+1+l > len(rdata) {
return rcode, answers, errShort
}
parts = append(parts, string(rdata[p+1:p+1+l]))
p += 1 + l
}
answers = append(answers, strings.Join(parts, ""))
}
}
return rcode, answers, nil
}
+187
View File
@@ -0,0 +1,187 @@
package main
import (
"encoding/binary"
"net"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode.
func respond(q []byte, rcode byte) []byte {
end := 12
for q[end] != 0 {
end += int(q[end]) + 1
}
question := q[12 : end+5]
name := strings.ToLower(string(q[13:end]))
r := append([]byte{}, q[:2]...)
r = append(r, 0x81, 0x80|rcode, 0, 1)
answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1
if answer {
r = append(r, 0, 1, 0, 0, 0, 0)
} else {
r = append(r, 0, 0, 0, 0, 0, 0)
}
r = append(r, question...)
if answer {
r = append(r, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1)
}
return r
}
func udpResolver(t *testing.T, rcode byte) string {
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { pc.Close() })
go func() {
buf := make([]byte, 512)
for {
n, from, err := pc.ReadFrom(buf)
if err != nil {
return
}
_, _ = pc.WriteTo(respond(buf[:n], rcode), from)
}
}()
return pc.LocalAddr().String()
}
func tcpResolver(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { l.Close() })
go func() {
for {
c, err := l.Accept()
if err != nil {
return
}
var size [2]byte
if _, err := readFull(c, size[:]); err == nil {
q := make([]byte, binary.BigEndian.Uint16(size[:]))
if _, err := readFull(c, q); err == nil {
r := respond(q, 0)
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
}
}
c.Close()
}
}()
return l.Addr().String()
}
// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer?
func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) {
got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" {
t.Fatalf("udp: %+v, %v", got, err)
}
got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second)
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" {
t.Fatalf("tcp: %+v, %v", got, err)
}
got, _ = AskDNS("nope.internal", "A", udpResolver(t, 3), false, time.Second)
if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 {
t.Fatalf("no such name is a result with its code: %+v", got)
}
got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second)
if got.Rcode != "REFUSED" || got.Error == "" {
t.Fatalf("a refusal: %+v", got)
}
}
func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) {
l, _ := net.Listen("tcp", "127.0.0.1:0")
addr := l.Addr().String()
l.Close()
got, err := AskDNS("novox.internal", "A", addr, true, time.Second)
if err != nil || !strings.Contains(got.Error, "connection refused") {
t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err)
}
if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil {
t.Fatal("a server given by name was accepted")
}
if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil {
t.Fatal("tcp without a server was accepted")
}
if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" {
t.Fatalf("default port: %s", a)
}
for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53",
"[::1]:53": "[::1]:53", "::1": "[::1]:53"} {
if a, err := serverAddress(server); err != nil || a != want {
t.Errorf("%q: %q, %v; want %q", server, a, err, want)
}
}
for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536",
"10.10.0.1:dns", ""} {
if a, err := serverAddress(server); err == nil {
t.Errorf("%q was taken as %q", server, a)
}
}
}
func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) {
root := t.TempDir()
must := func(err error) {
if err != nil {
t.Fatal(err)
}
}
must(os.MkdirAll(filepath.Join(root, "net"), 0o755))
head := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"
// 127.0.0.1:53 and 10.10.0.1:53 listening, a connection on 22 (not listening), and an udp 53 bound on loopback only.
must(os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(head+
" 0: 0100007F:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 111 1 0 100 0 0 10 0\n"+
" 1: 01000A0A:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 112 1 0 100 0 0 10 0\n"+
" 2: 0100007F:0016 0200007F:9C40 01 00000000:00000000 00:00000000 00000000 0 0 113 1 0 100 0 0 10 0\n"), 0o644))
must(os.WriteFile(filepath.Join(root, "net", "udp"), []byte(head+
" 0: 0100007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000 0 0 121 2 0 0\n"), 0o644))
must(os.MkdirAll(filepath.Join(root, "42", "fd"), 0o755))
must(os.WriteFile(filepath.Join(root, "42", "comm"), []byte("dnsmasq\n"), 0o644))
must(os.Symlink("socket:[111]", filepath.Join(root, "42", "fd", "5")))
got := Listening(root, 53)
if got.Count != 3 || len(got.Unread) != 0 {
t.Fatalf("%+v", got)
}
var where []string
for _, s := range got.Sockets {
where = append(where, s.Protocol+" "+s.Address)
}
if strings.Join(where, ",") != "tcp 10.10.0.1,tcp 127.0.0.1,udp 127.0.0.1" {
t.Fatalf("sockets: %v", where)
}
if got.Sockets[1].Process != "42/dnsmasq" || got.Sockets[0].Owner == "" || !strings.Contains(got.Owners, "2 socket(s)") {
t.Fatalf("owners: %+v", got)
}
if missing := Listening(t.TempDir(), 0); len(missing.Unread) != 2 {
t.Fatalf("an unreadable table is said, not left out: %+v", missing)
}
}
func TestListeningOnThisMachineFindsASocketItOpened(t *testing.T) {
if _, err := os.Stat("/proc/net/tcp"); err != nil {
t.Skip("no /proc here")
}
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
port := l.Addr().(*net.TCPAddr).Port
got := Listening("/proc", port)
if got.Count != 1 || got.Sockets[0].Address != "127.0.0.1" {
t.Fatalf("%+v", got)
}
if got.Sockets[0].Process == "" {
t.Fatalf("its own socket's program is not named: %+v", got)
}
}
+59 -2
View File
@@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"syscall"
"time"
)
@@ -50,8 +51,13 @@ func CheckTCP(host string, port int, timeout time.Duration) (TCPResult, error) {
// DNSResult is what netcheck_dns answers.
type DNSResult struct {
Name string `json:"name"`
Type string `json:"type"`
Name string `json:"name"`
Type string `json:"type"`
// Server and Protocol say which resolver was asked when one was named; empty, the machine's.
Server string `json:"server,omitempty"`
Protocol string `json:"protocol,omitempty"`
// Rcode is the named resolver's response code: NOERROR, NXDOMAIN, REFUSED, SERVFAIL…
Rcode string `json:"rcode,omitempty"`
Answers []string `json:"answers"`
ElapsedMS int64 `json:"elapsed_ms"`
Error string `json:"error,omitempty"`
@@ -64,6 +70,13 @@ var DNSTypes = []string{"A", "AAAA", "CNAME", "TXT", "MX"}
// the machine's /etc/resolv.conf and /etc/hosts, which is the resolver this machine's programs use.
// A name that does not resolve is a result with its error; an unknown type is refused.
func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
return AskDNS(name, kind, "", false, timeout)
}
// AskDNS is CheckDNS, or — with a server — the same question put to that resolver alone, over UDP or
// TCP, its hosts file and search domains left out (hq issue 348). No answer, a refusal and a response
// code other than NOERROR are results with their error; a malformed question is refused.
func AskDNS(name, kind, server string, overTCP bool, timeout time.Duration) (DNSResult, error) {
kind = strings.ToUpper(strings.TrimSpace(kind))
if kind == "" {
kind = "A"
@@ -76,6 +89,37 @@ func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
return DNSResult{}, fmt.Errorf("type %q is not one netcheck_dns looks up (%s)", kind, strings.Join(DNSTypes, ", "))
}
out := DNSResult{Name: name, Type: kind, Answers: []string{}}
if server != "" {
if _, err := serverAddress(server); err != nil {
return DNSResult{}, err
}
out.Server, out.Protocol = server, "udp"
if overTCP {
out.Protocol = "tcp"
}
start := time.Now()
rcode, answers, err := askServer(server, name, kind, overTCP, timeout)
out.ElapsedMS = time.Since(start).Milliseconds()
if err == nil {
out.Rcode = rcodeWords[rcode]
if out.Rcode == "" {
out.Rcode = "RCODE" + strconv.Itoa(rcode)
}
if rcode != 0 {
out.Error = "the resolver answered " + out.Rcode
}
}
out.Answers = append(out.Answers, answers...)
if err != nil {
out.Error = dnsWords(err, timeout)
}
if kind != "MX" {
sort.Strings(out.Answers)
}
return out, nil
} else if overTCP {
return DNSResult{}, errors.New("tcp asks a named server; give server too")
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
r := net.DefaultResolver
@@ -171,3 +215,16 @@ func timeoutOf(args map[string]any) (time.Duration, error) {
}
return d, nil
}
// dnsWords is a failed question in words: refused (nothing listens there), no answer in time, or what the
// system said.
func dnsWords(err error, timeout time.Duration) string {
var ne net.Error
switch {
case errors.Is(err, syscall.ECONNREFUSED):
return "connection refused: nothing listens on that address and port"
case errors.As(err, &ne) && ne.Timeout():
return fmt.Sprintf("no answer within %s", timeout)
}
return err.Error()
}
+2 -2
View File
@@ -57,12 +57,12 @@ func TestTimeoutIsDefaultedAndBounded(t *testing.T) {
}
}
func TestBothToolsAreListedUnprefixed(t *testing.T) {
func TestTheToolsAreListedUnprefixed(t *testing.T) {
names := map[string]bool{}
for _, tool := range tools() {
names[tool.Name] = true
}
if !names["netcheck_tcp"] || !names["netcheck_dns"] || len(names) != 2 {
if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 {
t.Fatalf("tools: %v", names)
}
}
+179
View File
@@ -0,0 +1,179 @@
package main
// What listens on this machine (hq issue 348): the kernel's own socket tables, read where `ss -lunt` reads
// them. During the 2026-10-09 outage the resolver's unit stayed active while it no longer listened on its
// private address, and nothing on the mesh could show that: whether a socket is bound is the kernel's to
// say, not the unit's.
import (
"bufio"
"encoding/hex"
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
)
// Socket is one listening TCP socket or one bound UDP socket.
type Socket struct {
Protocol string `json:"protocol"`
Address string `json:"address"`
Port int `json:"port"`
// Process is the program holding it, "pid/name"; empty when Owner says why it is not known.
Process string `json:"process,omitempty"`
Owner string `json:"owner,omitempty"`
inode string
}
// ListeningResult is what netcheck_listening answers.
type ListeningResult struct {
Sockets []Socket `json:"sockets"`
Count int `json:"count"`
// Unread names a table that could not be read: a socket in it is missing from the list, not absent.
Unread []string `json:"unread,omitempty"`
// Owners says how many sockets' programs are unknown and why, when any is.
Owners string `json:"owners,omitempty"`
}
// Listening reads the socket tables under root (the machine's /proc), keeps TCP sockets in the listen
// state and every bound UDP socket, on port when port is not 0.
func Listening(root string, port int) ListeningResult {
out := ListeningResult{Sockets: []Socket{}}
for _, t := range []struct{ file, protocol string }{
{"tcp", "tcp"}, {"tcp6", "tcp6"}, {"udp", "udp"}, {"udp6", "udp6"},
} {
f, err := os.Open(filepath.Join(root, "net", t.file))
if err != nil {
if !os.IsNotExist(err) || !strings.HasSuffix(t.file, "6") {
out.Unread = append(out.Unread, t.file+": "+err.Error())
}
continue
}
sockets, err := parseSocketTable(f, t.protocol, port)
f.Close()
if err != nil {
out.Unread = append(out.Unread, t.file+": "+err.Error())
}
out.Sockets = append(out.Sockets, sockets...)
}
unknown := owners(root, out.Sockets)
if unknown > 0 {
out.Owners = fmt.Sprintf("%d socket(s) without their program: held by a process of another account, whose "+
"open files this tool may not read; the address and port are the kernel's and complete", unknown)
}
sort.Slice(out.Sockets, func(i, j int) bool {
a, b := out.Sockets[i], out.Sockets[j]
if a.Port != b.Port {
return a.Port < b.Port
}
if a.Protocol != b.Protocol {
return a.Protocol < b.Protocol
}
return a.Address < b.Address
})
out.Count = len(out.Sockets)
return out
}
// parseSocketTable reads one /proc/net table: "sl local_address rem_address st … inode".
func parseSocketTable(f *os.File, protocol string, port int) ([]Socket, error) {
var out []Socket
sc := bufio.NewScanner(f)
first := true
for sc.Scan() {
if first {
first = false
continue
}
fields := strings.Fields(sc.Text())
if len(fields) < 10 {
continue
}
state := fields[3]
tcp := strings.HasPrefix(protocol, "tcp")
// TCP 0A is LISTEN; UDP 07 is a bound socket (unconnected), 01 a connected one, which is not listening.
if (tcp && state != "0A") || (!tcp && state != "07") {
continue
}
addr, p, err := hexAddress(fields[1])
if err != nil {
return out, err
}
if port != 0 && p != port {
continue
}
out = append(out, Socket{Protocol: protocol, Address: addr, Port: p, inode: fields[9]})
}
return out, sc.Err()
}
// hexAddress reads the kernel's "0100007F:0035": the address in host order per 32-bit word, the port big-endian.
func hexAddress(s string) (string, int, error) {
a, p, ok := strings.Cut(s, ":")
if !ok {
return "", 0, fmt.Errorf("address %q", s)
}
port, err := strconv.ParseUint(p, 16, 16)
if err != nil {
return "", 0, fmt.Errorf("port %q", p)
}
raw, err := hex.DecodeString(a)
if err != nil || (len(raw) != 4 && len(raw) != 16) {
return "", 0, fmt.Errorf("address %q", a)
}
ip := make(net.IP, len(raw))
for w := 0; w < len(raw); w += 4 {
ip[w], ip[w+1], ip[w+2], ip[w+3] = raw[w+3], raw[w+2], raw[w+1], raw[w]
}
return ip.String(), int(port), nil
}
// owners names each socket's program from the processes' open files, as far as they can be read, and
// counts those it could not name.
func owners(root string, sockets []Socket) int {
want := map[string][]int{}
for i, s := range sockets {
want["socket:["+s.inode+"]"] = append(want["socket:["+s.inode+"]"], i)
}
if len(want) == 0 {
return 0
}
procs, _ := os.ReadDir(root)
for _, p := range procs {
pid, err := strconv.Atoi(p.Name())
if err != nil {
continue
}
fds, err := os.ReadDir(filepath.Join(root, p.Name(), "fd"))
if err != nil {
continue
}
for _, fd := range fds {
link, err := os.Readlink(filepath.Join(root, p.Name(), "fd", fd.Name()))
if err != nil {
continue
}
idx, ok := want[link]
if !ok {
continue
}
name, _ := os.ReadFile(filepath.Join(root, p.Name(), "comm"))
for _, i := range idx {
if sockets[i].Process == "" {
sockets[i].Process = fmt.Sprintf("%d/%s", pid, strings.TrimSpace(string(name)))
}
}
}
}
unknown := 0
for i := range sockets {
if sockets[i].Process == "" {
sockets[i].Owner = "not readable by this tool's account"
unknown++
}
}
return unknown
}
+33 -3
View File
@@ -1,12 +1,14 @@
// netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches
// and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's
// own sockets and resolver — a TCP connect and a DNS lookup — and nothing that changes anything.
// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named),
// and what listens here — and nothing that changes anything.
// The module's HTTP check is its TypeScript bundle; the runtime serves both under one module.
package main
import (
"fmt"
"os"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -21,6 +23,29 @@ func main() {
func tools() []stdio.Tool {
return []stdio.Tool{
{
Name: "netcheck_listening",
Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " +
"from the kernel's own tables (what ss -lunt reads), with the program holding it where this tool may see it. " +
"port narrows it to one port, e.g. 53 to see on which addresses the resolver answers.",
Input: map[string]any{
"port": map[string]any{"type": "integer", "description": "only this port (default every port)"},
},
Run: func(args map[string]any) (any, error) {
port := 0
if _, given := args["port"]; given {
p, err := whole(args, "port", 0)
if err != nil {
return nil, err
}
if p < 1 || p > 65535 {
return nil, fmt.Errorf("port %d is not a port (1-65535)", p)
}
port = p
}
return Listening("/proc", port), nil
},
},
{
Name: "netcheck_tcp",
Description: "Check whether a TCP port is reachable from this machine: opens one connection " +
@@ -48,11 +73,14 @@ func tools() []stdio.Tool {
},
{
Name: "netcheck_dns",
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts). " +
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " +
"that one resolver alone over UDP (or TCP), answering its response code (NOERROR, NXDOMAIN, REFUSED…). " +
"type is A, AAAA, CNAME, TXT or MX; answers the records found, or the error.",
Input: map[string]any{
"name": map[string]any{"type": "string", "description": "the name to look up"},
"type": map[string]any{"type": "string", "enum": []string{"A", "AAAA", "CNAME", "TXT", "MX"}, "description": "record type (default A)"},
"server": map[string]any{"type": "string", "description": "ask this resolver alone: an address, with :port or without (53), e.g. 10.10.0.1"},
"tcp": map[string]any{"type": "boolean", "description": "ask the server over TCP rather than UDP (default false)"},
"timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 3000, at most 30000)"},
},
Run: func(args map[string]any) (any, error) {
@@ -65,7 +93,9 @@ func tools() []stdio.Tool {
if err != nil {
return nil, err
}
return CheckDNS(name, kind, timeout)
server, _ := args["server"].(string)
overTCP, _ := args["tcp"].(bool)
return AskDNS(name, kind, strings.TrimSpace(server), overTCP, timeout)
},
},
}
+13 -1
View File
@@ -4,8 +4,20 @@
"tools": [
"netcheck_tcp",
"netcheck_dns",
"netcheck_http"
"netcheck_http",
"netcheck_listening"
],
"replaces": {
"netcheck_listening": [
"ss -lunt",
"ss -ltn",
"netstat -lnt"
],
"netcheck_dns": [
"dig",
"nslookup"
]
},
"build": {
"artifacts": [
{