From 2b2397abe0e6676b8a8b1a52d4e99a3f2d057a8a Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:37:39 +0200 Subject: [PATCH 1/6] dnsmasq: ask the resolver a real question, and restart it when it stops answering (hq issue 348) On 2026-10-09 the control node's dnsmasq stayed active and running, its journal silent, and refused every question on its private address for twenty minutes; nothing in the module looked past the unit. The module is now Go (its TypeScript tools were never served): a root watch asks the machine's own name on every listen address over UDP and TCP every 15s, restarts the unit after two failing looks (2 min apart, 3 an hour), and says whether that helped. dnsmasq_health is the watch process's declared check, so a restart that did not help becomes the module's unhealthy condition; the service is judged by its unit. --- health-undeclared | 2 +- modules/dnsmasq/client.ts | 62 --- modules/dnsmasq/cmd/dnsmasq-tools/ask.go | 200 ++++++++++ .../dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go | 352 ++++++++++++++++++ modules/dnsmasq/cmd/dnsmasq-tools/look.go | 207 ++++++++++ modules/dnsmasq/cmd/dnsmasq-tools/main.go | 128 +++++++ modules/dnsmasq/cmd/dnsmasq-tools/tools.go | 160 ++++++++ modules/dnsmasq/cmd/dnsmasq-tools/watch.go | 216 +++++++++++ modules/dnsmasq/go.mod | 5 + modules/dnsmasq/go.sum | 2 + modules/dnsmasq/index.ts | 38 -- modules/dnsmasq/module.json | 52 ++- modules/dnsmasq/package.json | 9 - modules/dnsmasq/tools/index.ts | 31 -- modules/dnsmasq/tsconfig.json | 12 - 15 files changed, 1321 insertions(+), 155 deletions(-) delete mode 100644 modules/dnsmasq/client.ts create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/ask.go create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/look.go create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/main.go create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/tools.go create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/watch.go create mode 100644 modules/dnsmasq/go.mod create mode 100644 modules/dnsmasq/go.sum delete mode 100644 modules/dnsmasq/index.ts delete mode 100644 modules/dnsmasq/package.json delete mode 100644 modules/dnsmasq/tools/index.ts delete mode 100644 modules/dnsmasq/tsconfig.json diff --git a/health-undeclared b/health-undeclared index c69be7a1..4165b15c 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1,4 +1,4 @@ -70 +69 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running diff --git a/modules/dnsmasq/client.ts b/modules/dnsmasq/client.ts deleted file mode 100644 index ed6a5be1..00000000 --- a/modules/dnsmasq/client.ts +++ /dev/null @@ -1,62 +0,0 @@ -// dnsmasq's own code, living in the module (novox/hq ADR 0039). dnsmasq here answers the mesh's -// generated wildcard names (..) itself and forwards everything else to fixed -// upstreams — the predecessor's arrangement, so the machine and its containers ask it for the world. -// Its code reads what it was told to answer, and can resolve through itself to prove that it does. - -import { readFile } from "node:fs/promises"; -import { Resolver } from "node:dns/promises"; - -export interface AnsweredName { - /** A machine's internal name, e.g. "anchor.internal" — it and everything under it resolve here. */ - name: string; - address: string; -} - -export class DnsmasqClient { - constructor( - private readonly resolverPath: string, - private readonly address: string, - ) {} - - /** - * Build from the environment. Both values are node-local facts with mesh-chosen defaults — the - * wildcard file the module is sent, and the loopback address its config listens on (127.0.0.1, - * where the predecessor's resolver answered and every machine's resolv.conf already points) — so - * there is nothing to be unconfigured about; it never throws. - */ - static fromEnv(env: NodeJS.ProcessEnv = process.env): DnsmasqClient { - return new DnsmasqClient( - env.MESH_DNSMASQ_RESOLVER_PATH ?? "/etc/mesh-resolver/nodes.conf", - env.MESH_DNSMASQ_ADDRESS ?? "127.0.0.1", - ); - } - - /** The names this resolver answers, read from the mesh-generated wildcard file. */ - async answeredNames(): Promise { - let text: string; - try { - text = await readFile(this.resolverPath, "utf8"); - } catch { - return []; // not yet on the network, or the file has not been written — no names, not an error - } - const names: AnsweredName[] = []; - for (const line of text.split("\n")) { - // dnsmasq wildcard syntax the mesh writes: address=/./
. The file also - // carries local=//, which names no machine and is not matched here. - const match = line.match(/^address=\/([^/]+)\/(.+)$/); - if (match) names.push({ name: match[1], address: match[2] }); - } - return names; - } - - /** Resolve a name through this node's own resolver — the check that wildcard-resolution answers. */ - async resolve(name: string): Promise { - const resolver = new Resolver(); - resolver.setServers([this.address]); - try { - return await resolver.resolve4(name); - } catch { - return await resolver.resolve6(name); - } - } -} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/ask.go b/modules/dnsmasq/cmd/dnsmasq-tools/ask.go new file mode 100644 index 00000000..9bcc4c5a --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/ask.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math/rand/v2" + "net" + "strings" + "syscall" + "time" +) + +// A real question to the resolver, as any program on the machine asks it: one name, one record type, over +// UDP or TCP, to one address. Written out here rather than taken from a library: the module needs the +// header's answer code and the A records, nothing else, and Go's own resolver would read /etc/hosts and +// /etc/resolv.conf instead of asking the one address it is told (novox/hq issue 348). + +// Answer is what one question got. +type Answer struct { + // Server is where it was asked, and Over the transport. + Server string `json:"server"` + Over string `json:"over"` + Name string `json:"name"` + // Answered says a reply came; Code is its answer code (NOERROR, NXDOMAIN, …) and Addresses its A + // records. + Answered bool `json:"answered"` + Code string `json:"code,omitempty"` + Addresses []string `json:"addresses,omitempty"` + // Failed is why no reply came, in words: refused (nothing listens there), no answer within, or what + // the system said. + Failed string `json:"failed,omitempty"` + // Took is how long it took, in milliseconds. + Took int64 `json:"took_ms"` +} + +// rcodes are the answer codes by number. +var rcodes = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"} + +// query is one question for name's A record, with id. +func query(name string, id uint16) ([]byte, error) { + name = strings.TrimSuffix(name, ".") + if name == "" { + return nil, errors.New("no name to ask") + } + msg := make([]byte, 12, 64) + binary.BigEndian.PutUint16(msg[0:], id) + binary.BigEndian.PutUint16(msg[2:], 0x0100) // a standard query, recursion desired + binary.BigEndian.PutUint16(msg[4:], 1) // one question + for _, label := range strings.Split(name, ".") { + if label == "" || len(label) > 63 { + return nil, fmt.Errorf("%q is not a name that can be asked", name) + } + msg = append(msg, byte(len(label))) + msg = append(msg, label...) + } + msg = append(msg, 0, 0, 1, 0, 1) // root; type A; class IN + return msg, nil +} + +// readAnswer reads a reply to the question with id: its code and A records. +func readAnswer(msg []byte, id uint16) (string, []string, error) { + if len(msg) < 12 { + return "", nil, errors.New("a reply too short to be one") + } + if binary.BigEndian.Uint16(msg[0:]) != id { + return "", nil, errors.New("a reply to another question") + } + flags := binary.BigEndian.Uint16(msg[2:]) + code, ok := rcodes[int(flags&0xF)] + if !ok { + code = fmt.Sprintf("RCODE%d", flags&0xF) + } + qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:])) + at := 12 + for i := 0; i < qd; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return code, nil, err + } + at += 4 + } + var addresses []string + for i := 0; i < an; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return code, addresses, err + } + if at+10 > len(msg) { + return code, addresses, errors.New("a reply cut short") + } + kind, length := binary.BigEndian.Uint16(msg[at:]), int(binary.BigEndian.Uint16(msg[at+8:])) + at += 10 + if at+length > len(msg) { + return code, addresses, errors.New("a reply cut short") + } + if kind == 1 && length == 4 { + addresses = append(addresses, net.IP(msg[at:at+4]).String()) + } + at += length + } + return code, addresses, nil +} + +// skipName steps over a name in a message, a pointer ending it. +func skipName(msg []byte, at int) (int, error) { + for at < len(msg) { + n := int(msg[at]) + switch { + case n == 0: + return at + 1, nil + case n&0xC0 == 0xC0: + return at + 2, nil + } + at += n + 1 + } + return at, errors.New("a name runs past the reply") +} + +// Ask asks server (an address, port 53 unless given) for name's A record over "udp" or "tcp", within +// timeout. +func Ask(ctx context.Context, server, over, name string, timeout time.Duration) Answer { + if _, _, err := net.SplitHostPort(server); err != nil { + server = net.JoinHostPort(server, "53") + } + a := Answer{Server: server, Over: over, Name: name} + started := time.Now() + defer func() { a.Took = time.Since(started).Milliseconds() }() + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + id := uint16(rand.UintN(1 << 16)) + q, err := query(name, id) + if err != nil { + a.Failed = err.Error() + return a + } + conn, err := (&net.Dialer{}).DialContext(ctx, over, server) + if err != nil { + a.Failed = failedWords(err, timeout) + a.Took = time.Since(started).Milliseconds() + return a + } + defer conn.Close() + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + var reply []byte + if over == "tcp" { + framed := binary.BigEndian.AppendUint16(nil, uint16(len(q))) + if _, err = conn.Write(append(framed, q...)); err == nil { + head := make([]byte, 2) + if _, err = readFull(conn, head); err == nil { + reply = make([]byte, binary.BigEndian.Uint16(head)) + _, err = readFull(conn, reply) + } + } + } else if _, err = conn.Write(q); err == nil { + buf := make([]byte, 1500) + var n int + n, err = conn.Read(buf) + reply = buf[:n] + } + if err != nil { + a.Failed = failedWords(err, timeout) + a.Took = time.Since(started).Milliseconds() + return a + } + a.Answered = true + a.Code, a.Addresses, err = readAnswer(reply, id) + if err != nil { + a.Failed = err.Error() + } + a.Took = time.Since(started).Milliseconds() + return a +} + +func readFull(conn net.Conn, buf []byte) (int, error) { + read := 0 + for read < len(buf) { + n, err := conn.Read(buf[read:]) + read += n + if err != nil { + return read, err + } + } + return read, nil +} + +// failedWords is why no reply came: refused when nothing listens, no answer when nothing came in time. +func failedWords(err error, timeout time.Duration) string { + var timedOut interface{ Timeout() bool } + switch { + case errors.Is(err, syscall.ECONNREFUSED): + return "refused: nothing listens there" + case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timedOut) && timedOut.Timeout()): + return fmt.Sprintf("no answer within %s", timeout) + } + return err.Error() +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go new file mode 100644 index 00000000..197f0ed4 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go @@ -0,0 +1,352 @@ +package main + +import ( + "context" + "encoding/binary" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until +// stopped — as dnsmasq answers the machine's own name from the roster. +type standIn struct { + udp net.PacketConn + tcp net.Listener + wg sync.WaitGroup +} + +func reply(q []byte, address string) []byte { + // The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name. + out := append([]byte{}, q...) + binary.BigEndian.PutUint16(out[2:], 0x8180) + binary.BigEndian.PutUint16(out[6:], 1) + ip := net.ParseIP(address).To4() + out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4) + return append(out, ip...) +} + +func startStandIn(t *testing.T, at, port, address string) *standIn { + t.Helper() + s := &standIn{} + var err error + if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil { + t.Fatal(err) + } + if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil { + t.Fatal(err) + } + s.wg.Add(2) + go func() { + defer s.wg.Done() + buf := make([]byte, 1500) + for { + n, from, err := s.udp.ReadFrom(buf) + if err != nil { + return + } + _, _ = s.udp.WriteTo(reply(buf[:n], address), from) + } + }() + go func() { + defer s.wg.Done() + for { + c, err := s.tcp.Accept() + if err != nil { + return + } + head := make([]byte, 2) + if _, err := readFull(c, head); err == nil { + q := make([]byte, binary.BigEndian.Uint16(head)) + if _, err := readFull(c, q); err == nil { + r := reply(q, address) + _, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...)) + } + } + _ = c.Close() + } + }() + t.Cleanup(s.stop) + return s +} + +func (s *standIn) stop() { + _ = s.udp.Close() + _ = s.tcp.Close() + s.wg.Wait() +} + +// freePort is a port free on both loopback addresses the tests use. +func freePort(t *testing.T) string { + t.Helper() + for i := 0; i < 20; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + _ = l.Close() + ok := true + for _, at := range []string{"127.0.0.1", "127.0.0.2"} { + for _, over := range []string{"tcp", "udp"} { + if over == "tcp" { + if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil { + _ = c.Close() + } else { + ok = false + } + } else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil { + _ = c.Close() + } else { + ok = false + } + } + } + if ok { + return port + } + } + t.Fatal("no port free on both loopback addresses") + return "" +} + +// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address) +// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2. +func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine { + t.Helper() + root := t.TempDir() + write := func(path, text string) { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n") + write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+ + "address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n") + if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil { + t.Fatal(err) + } + return &Machine{Root: root, Ask: Ask, Now: now, Port: port} +} + +func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) { + listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n") + if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" { + t.Fatalf("listen addresses %v", listen) + } + names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" + + "host-record=novox.internal,10.77.0.1\n") + q, ok := Question(listen, names) + if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" { + t.Fatalf("asks %+v", q) + } + if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" { + t.Fatalf("with no own name in the roster, asks %+v", q) + } +} + +// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback, +// and refused every question on its private address. Its unit said healthy. The look does not. +func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) { + port := freePort(t) + m := aResolverMachine(t, port, time.Now) + private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" { + t.Fatalf("a resolver answering everywhere: %+v", l) + } + private.stop() + l := m.Look(t.Context()) + if l.Healthy || len(l.Wrong) != 2 { + t.Fatalf("a resolver that stopped answering on its private address: %+v", l) + } + if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") || + !strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") { + t.Fatalf("said as %q", why) + } +} + +func TestAWrongAnswerIsNotAnAnswer(t *testing.T) { + q := Named{Name: "novox.internal", Address: "10.77.0.1"} + for _, c := range []struct { + a Answer + want string + }{ + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""}, + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"}, + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"}, + {Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"}, + } { + if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v said %q, want %q", c.a, got, c.want) + } + } +} + +func failing(at time.Time) Look { + return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}}, + Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}} +} + +func healthy(at time.Time) Look { return Look{At: at, Healthy: true} } + +// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks +// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why. +func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + var s State + var why string + if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 { + t.Fatalf("one failing look restarted: %q", why) + } + if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { + t.Fatalf("a second failing look did not restart: %+v %q", s, why) + } + s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why}) + // It helped. + s, why = Step(s, healthy(t0.Add(2*LookEvery))) + if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped { + t.Fatalf("a healthy look after a restart: %+v %q", s, why) + } + // Failing again soon after: held for the gap, then restarted; and so on until the budget is spent. + at := t0.Add(3 * LookEvery) + restarts := 1 + for at.Add(LookEvery).Sub(t0) < 50*time.Minute { + at = at.Add(LookEvery) + s, why = Step(s, failing(at)) + if why == "" { + continue + } + last := s.Restarts[len(s.Restarts)-1].At + if at.Sub(last) < RestartGap { + t.Fatalf("restarted %s after the last", at.Sub(last)) + } + s.Restarts = append(s.Restarts, Restart{At: at, Why: why}) + restarts++ + } + if restarts != RestartsPerHour { + t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour) + } + if !strings.Contains(s.Held, "the most the watch makes") { + t.Fatalf("the spent budget is not said: %+v", s) + } + for _, r := range s.Restarts[1:] { + if r.Helped == nil || *r.Helped { + t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped) + } + } + // A look that could not ask — no configuration — restarts nothing. + var none State + broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}} + none, _ = Step(none, broken) + if _, why := Step(none, broken); why != "" { + t.Fatalf("a missing configuration restarted the resolver: %q", why) + } +} + +// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the +// watch notices on its second look, restarts the unit (which brings the listener back), says the restart +// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not +// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the +// operator. +func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + var mu sync.Mutex + now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock } + pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() } + m := aResolverMachine(t, port, now) + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") + private.stop() + + fixes := true + var asked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if fixes { + private = startStandIn(t, "127.0.0.2", port, "127.0.0.2") + } + return "", nil + } + var s State + look := func() { + t.Helper() + next, why := Step(s, m.Look(t.Context())) + if why != "" { + r := Restart{At: now().UTC(), Why: why} + if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil { + r.Error = err.Error() + } + next.Restarts = append(next.Restarts, r) + } + s = next + if err := m.WriteState(s); err != nil { + t.Fatal(err) + } + pass(LookEvery) + } + look() + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") { + t.Fatalf("after one failing look: %+v", h.Why) + } + look() + if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" { + t.Fatalf("restarted with %v", asked) + } + look() + if r := s.Restarts[0]; r.Helped == nil || !*r.Helped { + t.Fatalf("the restart that brought it back is said as %+v", r) + } + if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 { + t.Fatalf("after the restart helped: %+v", h) + } + + // It stops again, and this time a restart does not help. + private.stop() + fixes = false + pass(RestartGap) + for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ { + look() + } + h := m.Health(t.Context()) + if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") || + !strings.Contains(h.Why, "which did not help") { + t.Fatalf("a restart that did not help: %q", h.Why) + } + // And a watch that stopped looking is said, whatever the resolver does. + pass(Stale + time.Minute) + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") { + t.Fatalf("a stale watch: %q", h.Why) + } +} + +func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) { + var a Announcer + t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}} + if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 { + t.Fatalf("said at its start: %+v", got) + } + if got := a.Next(nil, false, old); len(got) != 0 { + t.Fatalf("an unreadable roster said %+v", got) + } + now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})} + got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now) + types := map[string]int{} + for _, e := range got { + types[e.Type]++ + } + if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 { + t.Fatalf("said %+v", got) + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/look.go b/modules/dnsmasq/cmd/dnsmasq-tools/look.go new file mode 100644 index 00000000..996664aa --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/look.go @@ -0,0 +1,207 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// A look asks the resolver one real question on every address its configuration tells it to answer on, +// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat +// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its +// journal silent, and refused every question on its private address for twenty minutes; its unit said +// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers. + +// The files a look reads, under the machine's root. +const ( + ConfigFile = "/etc/dnsmasq.conf" + NodesFile = "/etc/mesh-resolver/nodes.conf" +) + +// AskWithin bounds each question. +const AskWithin = 2 * time.Second + +// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock. +type Machine struct { + Root string + Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer + Now func() time.Time + // Port is where the resolver answers; 53 but in a test. + Port string +} + +// Here is this machine. +func Here() *Machine { + return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"} +} + +func (m *Machine) read(path string) (string, error) { + b, err := os.ReadFile(filepath.Join(m.Root, path)) + return string(b), err +} + +// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order. +func ListenAddresses(config string) []string { + var out []string + sc := bufio.NewScanner(strings.NewReader(config)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + value, ok := strings.CutPrefix(line, "listen-address=") + if !ok { + continue + } + for _, a := range strings.Split(value, ",") { + if a = strings.TrimSpace(a); a != "" { + out = append(out, a) + } + } + } + return out +} + +// Named is a machine's name and its private address, as the roster gave them to the resolver. +type Named struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address: +// its `host-record=,
` lines. +func AnsweredNames(nodes string) []Named { + var out []Named + sc := bufio.NewScanner(strings.NewReader(nodes)) + for sc.Scan() { + value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=") + if !ok { + continue + } + name, address, ok := strings.Cut(value, ",") + if ok && name != "" && address != "" { + out = append(out, Named{Name: name, Address: strings.TrimSpace(address)}) + } + } + return out +} + +// Question is the name a look asks, and the address a right answer carries: the machine's own when the +// roster names an address the resolver listens on, else the roster's first. +func Question(listen []string, names []Named) (Named, bool) { + for _, n := range names { + for _, a := range listen { + if n.Address == a { + return n, true + } + } + } + if len(names) > 0 { + return names[0], true + } + return Named{}, false +} + +// Look is one look: every listen address asked over both transports. +type Look struct { + At time.Time `json:"at"` + // Healthy says every question was answered rightly. + Healthy bool `json:"healthy"` + Asked Named `json:"asked"` + Answers []Answer `json:"answers"` + // Wrong is every question not answered rightly, in words; Problems what kept the look from asking. + Wrong []string `json:"wrong,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// Why is what is wrong in one sentence: empty when healthy. +func (l Look) Why() string { + if l.Healthy { + return "" + } + return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ") +} + +// Look asks every listen address its question, over UDP and TCP, all at once. +func (m *Machine) Look(ctx context.Context) Look { + l := Look{At: m.Now().UTC()} + config, err := m.read(ConfigFile) + if err != nil { + l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err)) + return l + } + listen := ListenAddresses(config) + if len(listen) == 0 { + l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile)) + return l + } + nodes, err := m.read(NodesFile) + if err != nil { + l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err)) + return l + } + q, ok := Question(listen, AnsweredNames(nodes)) + if !ok { + l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile)) + return l + } + l.Asked = q + var mu sync.Mutex + var wg sync.WaitGroup + for _, address := range listen { + for _, over := range []string{"udp", "tcp"} { + wg.Add(1) + go func(address, over string) { + defer wg.Done() + a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin) + mu.Lock() + l.Answers = append(l.Answers, a) + mu.Unlock() + }(address, over) + } + } + wg.Wait() + sort.Slice(l.Answers, func(i, j int) bool { + if l.Answers[i].Server != l.Answers[j].Server { + return l.Answers[i].Server < l.Answers[j].Server + } + return l.Answers[i].Over > l.Answers[j].Over + }) + for _, a := range l.Answers { + if w := wrongWords(a, q); w != "" { + l.Wrong = append(l.Wrong, w) + } + } + l.Healthy = len(l.Wrong) == 0 + return l +} + +// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the +// machine's address. +func wrongWords(a Answer, q Named) string { + at := a.Server + " over " + a.Over + switch { + case !a.Answered: + return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed) + case a.Code != "NOERROR": + return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code) + case a.Failed != "": + return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed) + } + for _, x := range a.Addresses { + if x == q.Address { + return "" + } + } + return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address) +} + +func orNothing(s []string) string { + if len(s) == 0 { + return "no address" + } + return strings.Join(s, ", ") +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/main.go b/modules/dnsmasq/cmd/dnsmasq-tools/main.go new file mode 100644 index 00000000..771d3ee5 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/main.go @@ -0,0 +1,128 @@ +// The dnsmasq module's Go bundle: one of the mesh's resolvers (novox/hq ADR 0194, 0223). +// +// Started with no arguments it is served by the node's runtime as the operator account, over MCP on stdio +// (ADR 0188, ADR 0193): the module's tools and its health, and beside them the events the resolver's +// answers give — a machine's name it now answers or no longer does, and a restart the watch made. Started +// as `dnsmasq-tools watch` it is the module's long-running process, as root: it asks the resolver a real +// question on every listen address and restarts it, within a budget, when it stops answering (watch.go, +// novox/hq issue 348). stdout is the MCP channel; everything else is said on stderr. +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func main() { + if len(os.Args) > 1 { + if os.Args[1] != "watch" || len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: dnsmasq-tools [watch]") + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) + defer stop() + m := Here() + if err := m.Watch(ctx, ExecRunner, m.WriteState); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + m := Here() + go Announce(m, stdio.Emit) + if err := stdio.Serve("", Tools(m)); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// AnnounceEvery is how often the bundle reads the roster and the watch for what to say. +var AnnounceEvery = 30 * time.Second + +// Announcer remembers what was said, primed silently so a restart of the bundle does not say every name +// and every restart again. +type Announcer struct { + names map[string]string + restarts map[time.Time]bool + // primed says the names were read once; heard that the watch's restarts were. + primed, heard bool +} + +// Event is one event to emit. +type Event struct { + Type string + Body map[string]any +} + +// Next is what changed since the last read: names the resolver now answers (`name.added`) or no longer +// does (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own +// events — and restarts the watch made (`restarted`), each with why and whether it helped once known. +// +// read says the roster was read: an unreadable roster says nothing about names, rather than every name +// removed. +func (a *Announcer) Next(names []Named, read bool, s State) []Event { + now := map[string]string{} + for _, n := range names { + now[n.Name] = n.Address + } + var out []Event + if a.primed && read { + for name, address := range now { + if _, had := a.names[name]; !had { + out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}}) + } + } + for name := range a.names { + if _, has := now[name]; !has { + out = append(out, Event{"name.removed", map[string]any{"name": name}}) + } + } + } + if a.restarts == nil { + a.restarts = map[time.Time]bool{} + } + for _, r := range s.Restarts { + if a.restarts[r.At] { + continue + } + a.restarts[r.At] = true + if a.heard { + body := map[string]any{"unit": Unit, "at": r.At, "why": r.Why} + if r.Error != "" { + body["error"] = r.Error + } + out = append(out, Event{"restarted", body}) + } + } + a.heard = true + if read { + a.names, a.primed = now, true + } + return out +} + +// Announce says the events for as long as the bundle runs. +func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) { + time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs + var a Announcer + for { + var names []Named + text, err := m.read(NodesFile) + if err == nil { + names = AnsweredNames(text) + } + s, _, _ := m.ReadState() + for _, e := range a.Next(names, err == nil, s) { + if err := emit(e.Type, e.Body); err != nil { + fmt.Fprintf(os.Stderr, "dnsmasq: %s not said: %v\n", e.Type, err) + } + } + time.Sleep(AnnounceEvery) + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go new file mode 100644 index 00000000..3d4d11e2 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Stale is how old the watch's last look may be before the health tool says it stopped looking. +const Stale = 2 * time.Minute + +// Health is the module's health answer (novox/hq ADR 0240): healthy only when the resolver answers its +// real question on every listen address, over UDP and TCP, and the watch is looking. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why,omitempty"` + Look Look `json:"look"` + // Watch is what the watch last saw and did; absent when it has written nothing yet. + Watch *State `json:"watch,omitempty"` +} + +// Health asks the resolver now, and reads what the watch did. +func (m *Machine) Health(ctx context.Context) Health { + l := m.Look(ctx) + h := Health{Healthy: l.Healthy, Look: l} + var why []string + if !l.Healthy { + why = append(why, "the resolver does not answer as it must: "+l.Why()) + } + s, had, err := m.ReadState() + switch { + case err != nil: + h.Healthy = false + why = append(why, "what the watch saw cannot be read: "+err.Error()) + case !had: + h.Healthy = false + why = append(why, "the watch has written nothing yet, so nothing restarts the resolver when it stops answering") + default: + h.Watch = &s + if age := m.Now().Sub(s.Last.At); age > Stale { + h.Healthy = false + why = append(why, fmt.Sprintf("the watch last looked %s ago, so nothing restarts the resolver when it "+ + "stops answering", age.Round(time.Second))) + } + if !l.Healthy { + why = append(why, restartWords(s)) + } + } + h.Why = strings.Join(nonEmpty(why), "; ") + return h +} + +// restartWords is what the watch did about it, in words. +func restartWords(s State) string { + if len(s.Restarts) == 0 { + return "the watch has not restarted it" + } + r := s.Restarts[len(s.Restarts)-1] + words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC")) + switch { + case r.Error != "": + words += ", and the restart failed: " + r.Error + case r.Helped != nil && !*r.Helped: + words += ", which did not help" + case r.Helped == nil: + words += ", and its next looks decide whether that helped" + } + if s.Held != "" { + words += "; " + s.Held + } + return words +} + +func nonEmpty(s []string) []string { + var out []string + for _, x := range s { + if x != "" { + out = append(out, x) + } + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func arg(a map[string]any, k string) string { + v, _ := a[k].(string) + return strings.TrimSpace(v) +} + +// Tools are the module's tools. +func Tools(m *Machine) []stdio.Tool { + return []stdio.Tool{ + {Name: "dnsmasq_health", Description: "Whether this machine's mesh resolver answers: its real question — this " + + "machine's own name — asked now on every address its configuration listens on, over UDP and TCP, each " + + "answer with its code, addresses and time; and what the module's watch last saw, the restarts it made " + + "within the hour, whether each helped, and why it holds back from another. Healthy only when every " + + "question was answered rightly and the watch is looking (novox/hq ADR 0240, issue 348). (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { + return m.Health(context.Background()), nil + }}, + {Name: "dnsmasq_names", Description: "The mesh's machine names this machine's resolver answers, each with its " + + "private address, as the roster wrote them (" + NodesFile + "). (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { + text, err := m.read(NodesFile) + if err != nil { + return nil, fmt.Errorf("the roster %s cannot be read: %w", NodesFile, err) + } + return map[string]any{"names": AnsweredNames(text)}, nil + }}, + {Name: "dnsmasq_resolve", Description: "Ask this machine's resolver one name's IPv4 address, directly, on one " + + "address it listens on (default every one) over udp or tcp (default both): the answer code, the addresses " + + "and the time each took, or why none came — refused when nothing listens there. Replaces dig @address. (r)", + Input: map[string]any{"name": str("the name to ask, e.g. plex.anchor.internal"), + "address": str("one listen address to ask (default: every one in its configuration)"), + "over": str("udp or tcp (default: both)")}, + Run: func(a map[string]any) (any, error) { + return m.Resolve(context.Background(), arg(a, "name"), arg(a, "address"), arg(a, "over")) + }}, + } +} + +// Resolve asks name on one listen address or every one, over one transport or both. +func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, error) { + if name == "" { + return nil, fmt.Errorf("name is required") + } + var addresses []string + if address != "" { + addresses = []string{address} + } else { + config, err := m.read(ConfigFile) + if err != nil { + return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + } + addresses = ListenAddresses(config) + } + overs := []string{"udp", "tcp"} + switch over { + case "": + case "udp", "tcp": + overs = []string{over} + default: + return nil, fmt.Errorf("over is udp or tcp, not %q", over) + } + var answers []Answer + for _, a := range addresses { + for _, o := range overs { + answers = append(answers, m.Ask(ctx, a+":"+m.Port, o, name, AskWithin)) + } + } + return map[string]any{"name": name, "answers": answers}, nil +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go new file mode 100644 index 00000000..4989dc8c --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go @@ -0,0 +1,216 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real +// question on every listen address (look.go); after Confirm failing looks in a row it restarts the +// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh +// (novox/hq issue 348). +// +// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that +// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor +// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where +// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked, +// and it never says it worked: its next look does. A restart that did not help is said by the health +// tool as the resolver's failure, which the node-engine states and the controller raises — the operator +// hears of it only then. + +// The watch's clocks and its budget. +var ( + LookEvery = 15 * time.Second + // Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277). + Confirm = 2 + // RestartGap is the least time between two restarts; RestartsPerHour the most within an hour. + RestartGap = 2 * time.Minute + RestartsPerHour = 3 + // HelpedWithin is how soon after a restart a healthy look says the restart helped. + HelpedWithin = time.Minute +) + +// Unit is the resolver's service. +const Unit = "dnsmasq.service" + +// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator +// account the tools run as, written only by the watch. +const StateFile = "/var/lib/dnsmasq-watch/watch.json" + +// Restart is one restart the watch made. +type Restart struct { + At time.Time `json:"at"` + Why string `json:"why"` + // Error is what the service manager said when the restart itself failed. + Error string `json:"error,omitempty"` + // Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once + // HelpedWithin passed without one, absent until then. + Helped *bool `json:"helped,omitempty"` +} + +// State is the watch's last look and what it did, as the tools read it. +type State struct { + Last Look `json:"last"` + // Failing is how many looks in a row failed; Since when the first of them looked. + Failing int `json:"failing"` + Since *time.Time `json:"since,omitempty"` + // Restarts are the restarts within the last hour and the last one ever, newest last. + Restarts []Restart `json:"restarts,omitempty"` + // Held is why a restart that was due was not made: the budget is spent. + Held string `json:"held,omitempty"` +} + +// Runner runs one command; injected, so the watch is tested over recorded answers. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// ExecRunner runs a command on the machine, bounded to a minute. +func ExecRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + var out bytes.Buffer + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return strings.TrimSpace(out.String()), err +} + +// Step is one look of the watch and what follows from it: the state after, and whether to restart and why. +// Pure but for the look it is given. +func Step(s State, l Look) (State, string) { + now := l.At + s.Last = l + // What the earlier restarts are now known to have done. + for i := range s.Restarts { + r := &s.Restarts[i] + if r.Helped != nil || r.Error != "" { + continue + } + switch { + case l.Healthy && now.Sub(r.At) <= HelpedWithin: + yes := true + r.Helped = &yes + case now.Sub(r.At) > HelpedWithin: + no := false + r.Helped = &no + } + } + // The budget's window: the last hour, and always the newest. + kept := s.Restarts[:0] + for i, r := range s.Restarts { + if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 { + kept = append(kept, r) + } + } + s.Restarts = kept + if l.Healthy { + s.Failing, s.Since, s.Held = 0, nil, "" + return s, "" + } + if s.Failing == 0 { + at := now + s.Since = &at + } + s.Failing++ + if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 { + // Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing + // configuration or roster is missing. + return s, "" + } + within := 0 + var newest *Restart + for i, r := range s.Restarts { + if now.Sub(r.At) <= time.Hour { + within++ + } + newest = &s.Restarts[i] + } + switch { + case newest != nil && now.Sub(newest.At) < RestartGap: + s.Held = "" + return s, "" // the last restart is still settling + case within >= RestartsPerHour: + s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+ + "more until the oldest is an hour old, and the resolver's failure is said as its health", within) + return s, "" + } + s.Held = "" + return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why()) +} + +// Watch runs the watch until ctx ends. +func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error { + var s State + if raw, err := m.read(StateFile); err == nil { + _ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart + } + tick := time.NewTicker(LookEvery) + defer tick.Stop() + for { + l := m.Look(ctx) + next, why := Step(s, l) + if why != "" { + r := Restart{At: m.Now().UTC(), Why: why} + if out, err := run(ctx, "systemctl", "restart", Unit); err != nil { + r.Error = strings.TrimSpace(err.Error() + " " + out) + } + fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error)) + next.Restarts = append(next.Restarts, r) + } else if !l.Healthy && next.Failing == Confirm { + fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held)) + } + s = next + if err := write(s); err != nil { + fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err) + } + select { + case <-ctx.Done(): + return nil + case <-tick.C: + } + } +} + +func orEmpty(prefix, s string) string { + if s == "" { + return "" + } + return prefix + s +} + +// WriteState writes the state whole, replacing the file in one rename. +func (m *Machine) WriteState(s State) error { + path := filepath.Join(m.Root, StateFile) + body, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + tmp := path + ".new" + if err := os.WriteFile(tmp, body, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// ReadState is the watch's last state, and false when it has written none. +func (m *Machine) ReadState() (State, bool, error) { + raw, err := m.read(StateFile) + if errors.Is(err, os.ErrNotExist) { + return State{}, false, nil + } + if err != nil { + return State{}, false, err + } + var s State + if err := json.Unmarshal([]byte(raw), &s); err != nil { + return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err) + } + return s, true, nil +} diff --git a/modules/dnsmasq/go.mod b/modules/dnsmasq/go.mod new file mode 100644 index 00000000..d439da63 --- /dev/null +++ b/modules/dnsmasq/go.mod @@ -0,0 +1,5 @@ +module dnsmasq + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/dnsmasq/go.sum b/modules/dnsmasq/go.sum new file mode 100644 index 00000000..b474419e --- /dev/null +++ b/modules/dnsmasq/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/dnsmasq/index.ts b/modules/dnsmasq/index.ts deleted file mode 100644 index d8327bdb..00000000 --- a/modules/dnsmasq/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -// dnsmasq's events. The resolver's answered set changes whenever the mesh rewrites the wildcard file -// and restarts it — a machine joined the private network or left it. This watches that file and -// announces, from the resolver's own vantage, that a name became (or stopped being) resolvable on -// this node: DNS having actually propagated here, distinct from the mesh's own node.* events. -// -// Emits (novox/hq ADR 0041/0042): -// module.dnsmasq.name.added — this node's resolver now answers a machine's name -// module.dnsmasq.name.removed — it no longer does - -import { emit } from "@novox/mesh-sdk/events"; -import { DnsmasqClient } from "./client.js"; - -const dnsmasq = DnsmasqClient.fromEnv(); - -// name -> address, primed silently so a restart does not re-announce every name it already answered. -const known = new Map(); -let primed = false; - -async function poll(): Promise { - const now = new Map((await dnsmasq.answeredNames()).map((a) => [a.name, a.address])); - if (primed) { - for (const [name, address] of now) { - if (!known.has(name)) await emit("name.added", { name, address }); - } - for (const [name] of known) { - if (!now.has(name)) await emit("name.removed", { name }); - } - } - known.clear(); - for (const [name, address] of now) known.set(name, address); - primed = true; -} - -const run = (): void => void poll().catch((err) => console.error(`[dnsmasq] ${err}`)); -setInterval(run, 30_000); -run(); - -console.log("[dnsmasq] watching the resolver's answered names"); diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 8fc98255..4469a297 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -17,7 +17,8 @@ ], "emits": [ "name.added", - "name.removed" + "name.removed", + "restarted" ], "own-secrets": { "broker": "${dir:mesh-state}/broker" @@ -28,6 +29,11 @@ "scope": "mesh" } ], + "tools": [ + "dnsmasq_health", + "dnsmasq_names", + "dnsmasq_resolve" + ], "listens": [ { "name": "dns-udp", @@ -75,7 +81,34 @@ "config", "dnsmasq.fact-node-zones", "dnsmasq.fact-zones" - ] + ], + "health": { + "kind": "unit" + } + }, + { + "id": "watch-dir", + "type": "directory", + "path": "/var/lib/dnsmasq-watch", + "mode": "0755" + }, + { + "id": "watch", + "type": "process", + "name": "dnsmasq-watch", + "artifact": "tools", + "run": [ + "./dnsmasq-tools", + "watch" + ], + "health": { + "kind": "tool", + "tool": "dnsmasq_health", + "interval": "20s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + } } ], "facts": { @@ -87,5 +120,20 @@ "path": "/etc/mesh-resolver/zones.conf", "template": "# Generated by the mesh. Do not edit — this file is replaced whenever a module declares a zone or\n# its machine moves (novox/hq ADR 0199). One forwarding line per zone, to the module answering it.\n\n{{range .Zones}}server=/{{.Zone}}/{{.Address}}#{{.Port}}\n{{end}}" } + }, + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/dnsmasq-tools", + "binary": "dnsmasq-tools", + "loads": [ + "dnsmasq-tools" + ] + } + ] } } diff --git a/modules/dnsmasq/package.json b/modules/dnsmasq/package.json deleted file mode 100644 index dbf68350..00000000 --- a/modules/dnsmasq/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@novox/module-dnsmasq", - "version": "0.1.0", - "description": "dnsmasq — the mesh's resolver: answers wildcard node names and forwards the rest upstream. Its tools and events live here.", - "type": "module", - "private": true, - "dependencies": { "@novox/mesh-sdk": "^0.1.0" }, - "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" } -} diff --git a/modules/dnsmasq/tools/index.ts b/modules/dnsmasq/tools/index.ts deleted file mode 100644 index bfeb7d8b..00000000 --- a/modules/dnsmasq/tools/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -// dnsmasq's tools — a resolver's two useful questions: what does it answer, and does it answer a -// given name. Moved into the module (novox/hq ADR 0039); the mesh serves them through the sdk. - -import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { DnsmasqClient } from "../client.js"; - -export function getDnsmasqTools(dnsmasq: DnsmasqClient): ToolDefinition[] { - return [ - { - name: "dnsmasq_names", - description: "The mesh names this node's resolver answers — each a machine and everything under it.", - input: {}, - run: async () => ({ names: await dnsmasq.answeredNames() }), - }, - { - name: "dnsmasq_resolve", - description: "Resolve a mesh name through this node's own resolver — the check that wildcard-resolution answers.", - input: { name: { type: "string", description: "a name to resolve, e.g. plex.anchor.internal" } }, - run: async (args) => { - const name = String(args.name); - try { - return { name, addresses: await dnsmasq.resolve(name) }; - } catch (err) { - return { name, addresses: [], error: err instanceof Error ? err.message : String(err) }; - } - }, - }, - ]; -} - -registerModuleTools("dnsmasq", (env) => getDnsmasqTools(DnsmasqClient.fromEnv(env))); diff --git a/modules/dnsmasq/tsconfig.json b/modules/dnsmasq/tsconfig.json deleted file mode 100644 index 36778592..00000000 --- a/modules/dnsmasq/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "noEmit": true - }, - "include": ["client.ts", "index.ts", "tools/index.ts"] -} From 3d8b52c9f42cd529cff8e4ad7a397397a436a7c3 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:24:58 +0200 Subject: [PATCH 2/6] docker: say where the runtime's event history begins, so a short history never reads as a quiet one (hq issue 348) docker_events asked for three hours across the 2026-10-09 resolver outage on the control node answered zero events: the runtime keeps only its last ~256 events in memory, and health-check execs filled them with the last ten minutes. The answer now carries held, oldest_held and complete, and a history line when the window reaches further back than what the runtime still holds. The type filter moves into the tool, since the history's start is only known from all of it. --- modules/docker/cmd/docker-tools/docker.go | 50 +++++++++++++++-- .../docker/cmd/docker-tools/docker_test.go | 53 ++++++++++++++++++- modules/docker/cmd/docker-tools/main.go | 3 +- 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/modules/docker/cmd/docker-tools/docker.go b/modules/docker/cmd/docker-tools/docker.go index 872b8f3d..7411ac21 100644 --- a/modules/docker/cmd/docker-tools/docker.go +++ b/modules/docker/cmd/docker-tools/docker.go @@ -991,16 +991,27 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st } // Events is what the runtime did in a window ending now, the latest last. +// +// **The runtime keeps only its last events in memory** (about 256), and the images' own health checks +// make three exec events each per look: on a machine with a few checked containers that history is ten +// minutes long or less, all of it exec events (hq research 032). A window asked for further back than +// the history reaches is answered with what is still held AND says where the history begins, so an +// empty list never reads as "nothing happened" (hq issue 348: three hours asked across an outage, +// answered zero). The type filter is applied here rather than by the runtime, for the same reason: the +// history's start is only known from all of it. func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int, execs bool) (map[string]any, error) { - args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"} + return c.eventsAt(ctx, time.Now(), minutes, kind, limit, execs) +} + +func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind string, limit int, execs bool) (map[string]any, error) { if kind != "" { switch kind { case "container", "image", "network", "volume", "daemon", "plugin", "builder": default: return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind) } - args = append(args, "--filter", "type="+kind) } + args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"} out, err := c.docker(ctx, args...) if err != nil { return nil, err @@ -1015,6 +1026,9 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int redacted := map[string]bool{} events := []map[string]any{} for _, e := range raw { + if kind != "" && e.Type != kind { + continue + } if !execs && strings.HasPrefix(e.Action, "exec_") { continue } @@ -1042,14 +1056,44 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int if len(events) > limit { events = events[len(events)-limit:] } - answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events} + answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events, + "held": len(raw)} if len(redacted) > 0 { answer["redacted"] = sortedSet(redacted) answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)" } + for k, v := range historyReach(raw, now, minutes) { + answer[k] = v + } return answer, nil } +// historyGap is how much later than the window's start the runtime's oldest held event may be and the +// window still read as covered: events are not made every second. +const historyGap = 2 * time.Minute + +// historyReach says where the runtime's history begins when it begins after the window asked for: the +// oldest event it still holds, and that nothing before it can be answered from here. +func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any { + start := now.Add(-time.Duration(minutes) * time.Minute) + if len(raw) == 0 { + return map[string]any{"complete": false, + "history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"} + } + oldest := time.Unix(0, raw[0].TimeNano) + for _, e := range raw { + if t := time.Unix(0, e.TimeNano); t.Before(oldest) { + oldest = t + } + } + if oldest.Sub(start) <= historyGap { + return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)} + } + return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339), + "history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)", + len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} +} + func sortedSet(set map[string]bool) []string { out := make([]string, 0, len(set)) for k := range set { diff --git a/modules/docker/cmd/docker-tools/docker_test.go b/modules/docker/cmd/docker-tools/docker_test.go index 22774e83..41ff8eb7 100644 --- a/modules/docker/cmd/docker-tools/docker_test.go +++ b/modules/docker/cmd/docker-tools/docker_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io/fs" "os" "reflect" @@ -307,7 +308,8 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" { t.Fatalf("%v", evs) } - if !f.ran("docker events --since 30m --until 0s --format {{json .}} --filter type=container") { + // The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348). + if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") { t.Fatalf("%+v", f.calls) } if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil { @@ -362,3 +364,52 @@ func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) { t.Fatalf("%v %v", u, err) } } + +// eventLine is one line of `docker events --format '{{json .}}'`. +func eventLine(at time.Time, kind, action, name string) string { + return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n", + kind, action, name, at.UnixNano()) +} + +// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the +// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened. +func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) { + now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) + var out strings.Builder + for i := 0; i < 250; i++ { + out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail")) + } + got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false) + if err != nil { + t.Fatal(err) + } + if got["count"] != 0 || got["held"] != 250 { + t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"]) + } + if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") || + got["oldest_held"] != "2026-10-09T11:14:00Z" { + t.Fatalf("an empty answer from a short history reads as quiet: %v", got) + } +} + +func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) { + now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) + out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + + eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") + + eventLine(now.Add(-1*time.Minute), "network", "create", "build-net") + f := (&fake{}).on("docker events", Ran{Stdout: out}) + got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false) + if err != nil { + t.Fatal(err) + } + if got["complete"] != true || got["count"] != 2 || got["history"] != nil { + t.Fatalf("a covered window: %v", got) + } + if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") { + t.Fatal("the type was filtered by the runtime, so the history's start is unknown") + } + got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) + if got["complete"] != false || got["history"] == nil { + t.Fatalf("no events at all reads as quiet: %v", got) + } +} diff --git a/modules/docker/cmd/docker-tools/main.go b/modules/docker/cmd/docker-tools/main.go index f5c812ea..9e95a39d 100644 --- a/modules/docker/cmd/docker-tools/main.go +++ b/modules/docker/cmd/docker-tools/main.go @@ -262,7 +262,8 @@ func tools(c *Client) []stdio.Tool { { Name: "docker_events", Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " + - "an exec's command line is shown with any secret it carried as [redacted: ] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag.", + "an exec's command line is shown with any secret it carried as [redacted: ] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag. " + + "The runtime keeps only its last ~256 events in memory, which health-check execs fill in minutes: complete=false and history say where what it holds begins, and an empty list then means gone, not quiet.", Input: map[string]any{ "minutes": map[string]any{"type": "integer", "description": "how far back (default 60, at most 1440)"}, "type": map[string]any{"type": "string", "description": "only one kind: container, image, network, volume, daemon, plugin or builder"}, From 0f6986aead5d61b9ce52f2aa7408d743ac302954 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:27:39 +0200 Subject: [PATCH 3/6] netcheck: ask one named resolver, and list what listens here (hq issue 348) During the 2026-10-09 outage the control node's resolver stayed active while it no longer listened on its private address, and nothing on the mesh could ask that address a question or show which sockets were bound. netcheck_dns takes server (and tcp) to ask one resolver alone and say its response code; netcheck_listening reads the kernel's socket tables, as ss -lunt does, with the owning program where the tool's account may see it and a sentence where it may not. --- modules/netcheck/cmd/netcheck/ask.go | 252 +++++++++++++++++++ modules/netcheck/cmd/netcheck/ask_test.go | 175 +++++++++++++ modules/netcheck/cmd/netcheck/checks.go | 60 ++++- modules/netcheck/cmd/netcheck/checks_test.go | 4 +- modules/netcheck/cmd/netcheck/listening.go | 179 +++++++++++++ modules/netcheck/cmd/netcheck/main.go | 36 ++- modules/netcheck/module.json | 14 +- 7 files changed, 712 insertions(+), 8 deletions(-) create mode 100644 modules/netcheck/cmd/netcheck/ask.go create mode 100644 modules/netcheck/cmd/netcheck/ask_test.go create mode 100644 modules/netcheck/cmd/netcheck/listening.go diff --git a/modules/netcheck/cmd/netcheck/ask.go b/modules/netcheck/cmd/netcheck/ask.go new file mode 100644 index 00000000..ade2f085 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/ask.go @@ -0,0 +1,252 @@ +package main + +// Asking one named resolver directly (hq issue 348). netcheck_dns without a server asks the machine's +// resolver the way its programs do — resolv.conf, hosts, search domains, several servers in turn — which +// answers "can this machine resolve?" and hides which resolver answered. During the 2026-10-09 outage the +// question was "does the resolver at 10.10.0.1 answer?", and nothing on the mesh could ask it. Asked with a +// server, the question goes to that address and port alone, over UDP or TCP, and the answer says its +// response code: a refusal, no such name, and no answer at all are three different findings. +// +// The wire format is written here rather than taken from a library: a question and the records of five +// types are all it needs, and the module stays without a dependency. + +import ( + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" +) + +var dnsTypeCodes = map[string]uint16{"A": 1, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28} + +var rcodeWords = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"} + +// serverAddress is a server as asked — an address, with a port or without (53) — as host:port. +func serverAddress(server string) (string, error) { + server = strings.TrimSpace(server) + if host, port, err := net.SplitHostPort(server); err == nil { + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return "", fmt.Errorf("server %q: the port is not one (1-65535)", server) + } + if net.ParseIP(host) == nil { + return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server) + } + return net.JoinHostPort(host, port), nil + } + if net.ParseIP(server) == nil { + return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server) + } + return net.JoinHostPort(server, "53"), nil +} + +// askServer asks one resolver one question and reads its answer. +func askServer(server, name, kind string, overTCP bool, timeout time.Duration) (rcode int, answers []string, err error) { + addr, err := serverAddress(server) + if err != nil { + return 0, nil, err + } + query, id, err := buildQuery(name, dnsTypeCodes[kind]) + if err != nil { + return 0, nil, err + } + network := "udp" + if overTCP { + network = "tcp" + } + conn, err := net.DialTimeout(network, addr, timeout) + if err != nil { + return 0, nil, err + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(timeout)) + var reply []byte + if overTCP { + framed := binary.BigEndian.AppendUint16(nil, uint16(len(query))) + if _, err := conn.Write(append(framed, query...)); err != nil { + return 0, nil, err + } + var size [2]byte + if _, err := readFull(conn, size[:]); err != nil { + return 0, nil, err + } + reply = make([]byte, binary.BigEndian.Uint16(size[:])) + if _, err := readFull(conn, reply); err != nil { + return 0, nil, err + } + } else { + if _, err := conn.Write(query); err != nil { + return 0, nil, err + } + buf := make([]byte, 4096) + n, err := conn.Read(buf) + if err != nil { + return 0, nil, err + } + reply = buf[:n] + } + return parseReply(reply, id, dnsTypeCodes[kind]) +} + +func readFull(conn net.Conn, b []byte) (int, error) { + got := 0 + for got < len(b) { + n, err := conn.Read(b[got:]) + got += n + if err != nil { + return got, err + } + } + return got, nil +} + +// buildQuery is one question, recursion desired, under a random id. +func buildQuery(name string, qtype uint16) ([]byte, uint16, error) { + var idb [2]byte + _, _ = rand.Read(idb[:]) + id := binary.BigEndian.Uint16(idb[:]) + msg := binary.BigEndian.AppendUint16(nil, id) + msg = append(msg, 0x01, 0x00) // recursion desired + msg = append(msg, 0, 1, 0, 0, 0, 0, 0, 0) // one question + wire, err := encodeName(name) + if err != nil { + return nil, 0, err + } + msg = append(msg, wire...) + msg = binary.BigEndian.AppendUint16(msg, qtype) + msg = binary.BigEndian.AppendUint16(msg, 1) // IN + return msg, id, nil +} + +func encodeName(name string) ([]byte, error) { + name = strings.TrimSuffix(strings.TrimSpace(name), ".") + var out []byte + if name != "" { + for _, label := range strings.Split(name, ".") { + if label == "" || len(label) > 63 { + return nil, fmt.Errorf("name %q has an empty or too long label", name) + } + out = append(out, byte(len(label))) + out = append(out, label...) + } + } + return append(out, 0), nil +} + +var errShort = errors.New("the answer is cut short") + +// readName reads a possibly compressed name at off, and the offset after it. +func readName(msg []byte, off int) (string, int, error) { + var labels []string + next, jumped := -1, 0 + for { + if off >= len(msg) { + return "", 0, errShort + } + l := int(msg[off]) + switch { + case l == 0: + if next < 0 { + next = off + 1 + } + return strings.Join(labels, "."), next, nil + case l&0xC0 == 0xC0: + if off+1 >= len(msg) { + return "", 0, errShort + } + if next < 0 { + next = off + 2 + } + if jumped++; jumped > 32 { + return "", 0, errors.New("the answer's names point in a loop") + } + off = int(binary.BigEndian.Uint16(msg[off:]) & 0x3FFF) + default: + if off+1+l > len(msg) { + return "", 0, errShort + } + labels = append(labels, string(msg[off+1:off+1+l])) + off += 1 + l + } + } +} + +// parseReply reads the response code and the answers of the asked type. +func parseReply(msg []byte, id, qtype uint16) (int, []string, error) { + if len(msg) < 12 { + return 0, nil, errShort + } + if binary.BigEndian.Uint16(msg) != id { + return 0, nil, errors.New("the answer is to another question (its id differs)") + } + if msg[2]&0x80 == 0 { + return 0, nil, errors.New("the answer is not marked as one") + } + rcode := int(msg[3] & 0x0F) + qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:])) + off := 12 + for i := 0; i < qd; i++ { + _, n, err := readName(msg, off) + if err != nil { + return rcode, nil, err + } + off = n + 4 + } + answers := []string{} + for i := 0; i < an; i++ { + _, n, err := readName(msg, off) + if err != nil { + return rcode, answers, err + } + off = n + if off+10 > len(msg) { + return rcode, answers, errShort + } + rtype := binary.BigEndian.Uint16(msg[off:]) + size := int(binary.BigEndian.Uint16(msg[off+8:])) + data := off + 10 + if data+size > len(msg) { + return rcode, answers, errShort + } + rdata := msg[data : data+size] + off = data + size + if rtype != qtype { + continue // a CNAME on the way to an address, say + } + switch rtype { + case 1, 28: + answers = append(answers, net.IP(rdata).String()) + case 5: + s, _, err := readName(msg, data) + if err != nil { + return rcode, answers, err + } + answers = append(answers, s) + case 15: + if size < 3 { + return rcode, answers, errShort + } + s, _, err := readName(msg, data+2) + if err != nil { + return rcode, answers, err + } + answers = append(answers, fmt.Sprintf("%d %s", binary.BigEndian.Uint16(rdata), s)) + case 16: + var parts []string + for p := 0; p < len(rdata); { + l := int(rdata[p]) + if p+1+l > len(rdata) { + return rcode, answers, errShort + } + parts = append(parts, string(rdata[p+1:p+1+l])) + p += 1 + l + } + answers = append(answers, strings.Join(parts, "")) + } + } + return rcode, answers, nil +} diff --git a/modules/netcheck/cmd/netcheck/ask_test.go b/modules/netcheck/cmd/netcheck/ask_test.go new file mode 100644 index 00000000..19ca738a --- /dev/null +++ b/modules/netcheck/cmd/netcheck/ask_test.go @@ -0,0 +1,175 @@ +package main + +import ( + "encoding/binary" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode. +func respond(q []byte, rcode byte) []byte { + end := 12 + for q[end] != 0 { + end += int(q[end]) + 1 + } + question := q[12 : end+5] + name := strings.ToLower(string(q[13:end])) + r := append([]byte{}, q[:2]...) + r = append(r, 0x81, 0x80|rcode, 0, 1) + answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1 + if answer { + r = append(r, 0, 1, 0, 0, 0, 0) + } else { + r = append(r, 0, 0, 0, 0, 0, 0) + } + r = append(r, question...) + if answer { + r = append(r, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1) + } + return r +} + +func udpResolver(t *testing.T, rcode byte) string { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + go func() { + buf := make([]byte, 512) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + _, _ = pc.WriteTo(respond(buf[:n], rcode), from) + } + }() + return pc.LocalAddr().String() +} + +func tcpResolver(t *testing.T) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + var size [2]byte + if _, err := readFull(c, size[:]); err == nil { + q := make([]byte, binary.BigEndian.Uint16(size[:])) + if _, err := readFull(c, q); err == nil { + r := respond(q, 0) + _, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...)) + } + } + c.Close() + } + }() + return l.Addr().String() +} + +// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer? +func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) { + got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second) + if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" { + t.Fatalf("udp: %+v, %v", got, err) + } + got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second) + if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" { + t.Fatalf("tcp: %+v, %v", got, err) + } + got, _ = AskDNS("nope.internal", "A", udpResolver(t, 3), false, time.Second) + if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 { + t.Fatalf("no such name is a result with its code: %+v", got) + } + got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second) + if got.Rcode != "REFUSED" || got.Error == "" { + t.Fatalf("a refusal: %+v", got) + } +} + +func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) { + l, _ := net.Listen("tcp", "127.0.0.1:0") + addr := l.Addr().String() + l.Close() + got, err := AskDNS("novox.internal", "A", addr, true, time.Second) + if err != nil || !strings.Contains(got.Error, "connection refused") { + t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err) + } + if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil { + t.Fatal("a server given by name was accepted") + } + if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil { + t.Fatal("tcp without a server was accepted") + } + if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" { + t.Fatalf("default port: %s", a) + } +} + +func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) { + root := t.TempDir() + must := func(err error) { + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(root, "net"), 0o755)) + head := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + // 127.0.0.1:53 and 10.10.0.1:53 listening, a connection on 22 (not listening), and an udp 53 bound on loopback only. + must(os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(head+ + " 0: 0100007F:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 111 1 0 100 0 0 10 0\n"+ + " 1: 01000A0A:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 112 1 0 100 0 0 10 0\n"+ + " 2: 0100007F:0016 0200007F:9C40 01 00000000:00000000 00:00000000 00000000 0 0 113 1 0 100 0 0 10 0\n"), 0o644)) + must(os.WriteFile(filepath.Join(root, "net", "udp"), []byte(head+ + " 0: 0100007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000 0 0 121 2 0 0\n"), 0o644)) + must(os.MkdirAll(filepath.Join(root, "42", "fd"), 0o755)) + must(os.WriteFile(filepath.Join(root, "42", "comm"), []byte("dnsmasq\n"), 0o644)) + must(os.Symlink("socket:[111]", filepath.Join(root, "42", "fd", "5"))) + got := Listening(root, 53) + if got.Count != 3 || len(got.Unread) != 0 { + t.Fatalf("%+v", got) + } + var where []string + for _, s := range got.Sockets { + where = append(where, s.Protocol+" "+s.Address) + } + if strings.Join(where, ",") != "tcp 10.10.0.1,tcp 127.0.0.1,udp 127.0.0.1" { + t.Fatalf("sockets: %v", where) + } + if got.Sockets[1].Process != "42/dnsmasq" || got.Sockets[0].Owner == "" || !strings.Contains(got.Owners, "2 socket(s)") { + t.Fatalf("owners: %+v", got) + } + if missing := Listening(t.TempDir(), 0); len(missing.Unread) != 2 { + t.Fatalf("an unreadable table is said, not left out: %+v", missing) + } +} + +func TestListeningOnThisMachineFindsASocketItOpened(t *testing.T) { + if _, err := os.Stat("/proc/net/tcp"); err != nil { + t.Skip("no /proc here") + } + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + port := l.Addr().(*net.TCPAddr).Port + got := Listening("/proc", port) + if got.Count != 1 || got.Sockets[0].Address != "127.0.0.1" { + t.Fatalf("%+v", got) + } + if got.Sockets[0].Process == "" { + t.Fatalf("its own socket's program is not named: %+v", got) + } +} diff --git a/modules/netcheck/cmd/netcheck/checks.go b/modules/netcheck/cmd/netcheck/checks.go index 152b6e43..8fbf74a9 100644 --- a/modules/netcheck/cmd/netcheck/checks.go +++ b/modules/netcheck/cmd/netcheck/checks.go @@ -50,8 +50,13 @@ func CheckTCP(host string, port int, timeout time.Duration) (TCPResult, error) { // DNSResult is what netcheck_dns answers. type DNSResult struct { - Name string `json:"name"` - Type string `json:"type"` + Name string `json:"name"` + Type string `json:"type"` + // Server and Protocol say which resolver was asked when one was named; empty, the machine's. + Server string `json:"server,omitempty"` + Protocol string `json:"protocol,omitempty"` + // Rcode is the named resolver's response code: NOERROR, NXDOMAIN, REFUSED, SERVFAIL… + Rcode string `json:"rcode,omitempty"` Answers []string `json:"answers"` ElapsedMS int64 `json:"elapsed_ms"` Error string `json:"error,omitempty"` @@ -64,6 +69,13 @@ var DNSTypes = []string{"A", "AAAA", "CNAME", "TXT", "MX"} // the machine's /etc/resolv.conf and /etc/hosts, which is the resolver this machine's programs use. // A name that does not resolve is a result with its error; an unknown type is refused. func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) { + return AskDNS(name, kind, "", false, timeout) +} + +// AskDNS is CheckDNS, or — with a server — the same question put to that resolver alone, over UDP or +// TCP, its hosts file and search domains left out (hq issue 348). No answer, a refusal and a response +// code other than NOERROR are results with their error; a malformed question is refused. +func AskDNS(name, kind, server string, overTCP bool, timeout time.Duration) (DNSResult, error) { kind = strings.ToUpper(strings.TrimSpace(kind)) if kind == "" { kind = "A" @@ -76,6 +88,37 @@ func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) { return DNSResult{}, fmt.Errorf("type %q is not one netcheck_dns looks up (%s)", kind, strings.Join(DNSTypes, ", ")) } out := DNSResult{Name: name, Type: kind, Answers: []string{}} + if server != "" { + if _, err := serverAddress(server); err != nil { + return DNSResult{}, err + } + out.Server, out.Protocol = server, "udp" + if overTCP { + out.Protocol = "tcp" + } + start := time.Now() + rcode, answers, err := askServer(server, name, kind, overTCP, timeout) + out.ElapsedMS = time.Since(start).Milliseconds() + if err == nil { + out.Rcode = rcodeWords[rcode] + if out.Rcode == "" { + out.Rcode = "RCODE" + strconv.Itoa(rcode) + } + if rcode != 0 { + out.Error = "the resolver answered " + out.Rcode + } + } + out.Answers = append(out.Answers, answers...) + if err != nil { + out.Error = dnsWords(err, timeout) + } + if kind != "MX" { + sort.Strings(out.Answers) + } + return out, nil + } else if overTCP { + return DNSResult{}, errors.New("tcp asks a named server; give server too") + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() r := net.DefaultResolver @@ -171,3 +214,16 @@ func timeoutOf(args map[string]any) (time.Duration, error) { } return d, nil } + +// dnsWords is a failed question in words: refused (nothing listens there), no answer in time, or what the +// system said. +func dnsWords(err error, timeout time.Duration) string { + var ne net.Error + switch { + case strings.Contains(err.Error(), "connection refused"): + return "connection refused: nothing listens on that address and port" + case errors.As(err, &ne) && ne.Timeout(): + return fmt.Sprintf("no answer within %s", timeout) + } + return err.Error() +} diff --git a/modules/netcheck/cmd/netcheck/checks_test.go b/modules/netcheck/cmd/netcheck/checks_test.go index 8e2af867..fb770c77 100644 --- a/modules/netcheck/cmd/netcheck/checks_test.go +++ b/modules/netcheck/cmd/netcheck/checks_test.go @@ -57,12 +57,12 @@ func TestTimeoutIsDefaultedAndBounded(t *testing.T) { } } -func TestBothToolsAreListedUnprefixed(t *testing.T) { +func TestTheToolsAreListedUnprefixed(t *testing.T) { names := map[string]bool{} for _, tool := range tools() { names[tool.Name] = true } - if !names["netcheck_tcp"] || !names["netcheck_dns"] || len(names) != 2 { + if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 { t.Fatalf("tools: %v", names) } } diff --git a/modules/netcheck/cmd/netcheck/listening.go b/modules/netcheck/cmd/netcheck/listening.go new file mode 100644 index 00000000..cc5a8ac2 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/listening.go @@ -0,0 +1,179 @@ +package main + +// What listens on this machine (hq issue 348): the kernel's own socket tables, read where `ss -lunt` reads +// them. During the 2026-10-09 outage the resolver's unit stayed active while it no longer listened on its +// private address, and nothing on the mesh could show that: whether a socket is bound is the kernel's to +// say, not the unit's. + +import ( + "bufio" + "encoding/hex" + "fmt" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Socket is one listening TCP socket or one bound UDP socket. +type Socket struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + // Process is the program holding it, "pid/name"; empty when Owner says why it is not known. + Process string `json:"process,omitempty"` + Owner string `json:"owner,omitempty"` + inode string +} + +// ListeningResult is what netcheck_listening answers. +type ListeningResult struct { + Sockets []Socket `json:"sockets"` + Count int `json:"count"` + // Unread names a table that could not be read: a socket in it is missing from the list, not absent. + Unread []string `json:"unread,omitempty"` + // Owners says how many sockets' programs are unknown and why, when any is. + Owners string `json:"owners,omitempty"` +} + +// Listening reads the socket tables under root (the machine's /proc), keeps TCP sockets in the listen +// state and every bound UDP socket, on port when port is not 0. +func Listening(root string, port int) ListeningResult { + out := ListeningResult{Sockets: []Socket{}} + for _, t := range []struct{ file, protocol string }{ + {"tcp", "tcp"}, {"tcp6", "tcp6"}, {"udp", "udp"}, {"udp6", "udp6"}, + } { + f, err := os.Open(filepath.Join(root, "net", t.file)) + if err != nil { + if !os.IsNotExist(err) || !strings.HasSuffix(t.file, "6") { + out.Unread = append(out.Unread, t.file+": "+err.Error()) + } + continue + } + sockets, err := parseSocketTable(f, t.protocol, port) + f.Close() + if err != nil { + out.Unread = append(out.Unread, t.file+": "+err.Error()) + } + out.Sockets = append(out.Sockets, sockets...) + } + unknown := owners(root, out.Sockets) + if unknown > 0 { + out.Owners = fmt.Sprintf("%d socket(s) without their program: held by a process of another account, whose "+ + "open files this tool may not read; the address and port are the kernel's and complete", unknown) + } + sort.Slice(out.Sockets, func(i, j int) bool { + a, b := out.Sockets[i], out.Sockets[j] + if a.Port != b.Port { + return a.Port < b.Port + } + if a.Protocol != b.Protocol { + return a.Protocol < b.Protocol + } + return a.Address < b.Address + }) + out.Count = len(out.Sockets) + return out +} + +// parseSocketTable reads one /proc/net table: "sl local_address rem_address st … inode". +func parseSocketTable(f *os.File, protocol string, port int) ([]Socket, error) { + var out []Socket + sc := bufio.NewScanner(f) + first := true + for sc.Scan() { + if first { + first = false + continue + } + fields := strings.Fields(sc.Text()) + if len(fields) < 10 { + continue + } + state := fields[3] + tcp := strings.HasPrefix(protocol, "tcp") + // TCP 0A is LISTEN; UDP 07 is a bound socket (unconnected), 01 a connected one, which is not listening. + if (tcp && state != "0A") || (!tcp && state != "07") { + continue + } + addr, p, err := hexAddress(fields[1]) + if err != nil { + return out, err + } + if port != 0 && p != port { + continue + } + out = append(out, Socket{Protocol: protocol, Address: addr, Port: p, inode: fields[9]}) + } + return out, sc.Err() +} + +// hexAddress reads the kernel's "0100007F:0035": the address in host order per 32-bit word, the port big-endian. +func hexAddress(s string) (string, int, error) { + a, p, ok := strings.Cut(s, ":") + if !ok { + return "", 0, fmt.Errorf("address %q", s) + } + port, err := strconv.ParseUint(p, 16, 16) + if err != nil { + return "", 0, fmt.Errorf("port %q", p) + } + raw, err := hex.DecodeString(a) + if err != nil || (len(raw) != 4 && len(raw) != 16) { + return "", 0, fmt.Errorf("address %q", a) + } + ip := make(net.IP, len(raw)) + for w := 0; w < len(raw); w += 4 { + ip[w], ip[w+1], ip[w+2], ip[w+3] = raw[w+3], raw[w+2], raw[w+1], raw[w] + } + return ip.String(), int(port), nil +} + +// owners names each socket's program from the processes' open files, as far as they can be read, and +// counts those it could not name. +func owners(root string, sockets []Socket) int { + want := map[string][]int{} + for i, s := range sockets { + want["socket:["+s.inode+"]"] = append(want["socket:["+s.inode+"]"], i) + } + if len(want) == 0 { + return 0 + } + procs, _ := os.ReadDir(root) + for _, p := range procs { + pid, err := strconv.Atoi(p.Name()) + if err != nil { + continue + } + fds, err := os.ReadDir(filepath.Join(root, p.Name(), "fd")) + if err != nil { + continue + } + for _, fd := range fds { + link, err := os.Readlink(filepath.Join(root, p.Name(), "fd", fd.Name())) + if err != nil { + continue + } + idx, ok := want[link] + if !ok { + continue + } + name, _ := os.ReadFile(filepath.Join(root, p.Name(), "comm")) + for _, i := range idx { + if sockets[i].Process == "" { + sockets[i].Process = fmt.Sprintf("%d/%s", pid, strings.TrimSpace(string(name))) + } + } + } + } + unknown := 0 + for i := range sockets { + if sockets[i].Process == "" { + sockets[i].Owner = "not readable by this tool's account" + unknown++ + } + } + return unknown +} diff --git a/modules/netcheck/cmd/netcheck/main.go b/modules/netcheck/cmd/netcheck/main.go index 98cb5074..99f14075 100644 --- a/modules/netcheck/cmd/netcheck/main.go +++ b/modules/netcheck/cmd/netcheck/main.go @@ -1,12 +1,14 @@ // netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches // and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's -// own sockets and resolver — a TCP connect and a DNS lookup — and nothing that changes anything. +// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named), +// and what listens here — and nothing that changes anything. // The module's HTTP check is its TypeScript bundle; the runtime serves both under one module. package main import ( "fmt" "os" + "strings" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -21,6 +23,29 @@ func main() { func tools() []stdio.Tool { return []stdio.Tool{ + { + Name: "netcheck_listening", + Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " + + "from the kernel's own tables (what ss -lunt reads), with the program holding it where this tool may see it. " + + "port narrows it to one port, e.g. 53 to see on which addresses the resolver answers.", + Input: map[string]any{ + "port": map[string]any{"type": "integer", "description": "only this port (default every port)"}, + }, + Run: func(args map[string]any) (any, error) { + port := 0 + if _, given := args["port"]; given { + p, err := whole(args, "port", 0) + if err != nil { + return nil, err + } + if p < 1 || p > 65535 { + return nil, fmt.Errorf("port %d is not a port (1-65535)", p) + } + port = p + } + return Listening("/proc", port), nil + }, + }, { Name: "netcheck_tcp", Description: "Check whether a TCP port is reachable from this machine: opens one connection " + @@ -48,11 +73,14 @@ func tools() []stdio.Tool { }, { Name: "netcheck_dns", - Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts). " + + Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " + + "that one resolver alone over UDP (or TCP), answering its response code (NOERROR, NXDOMAIN, REFUSED…). " + "type is A, AAAA, CNAME, TXT or MX; answers the records found, or the error.", Input: map[string]any{ "name": map[string]any{"type": "string", "description": "the name to look up"}, "type": map[string]any{"type": "string", "enum": []string{"A", "AAAA", "CNAME", "TXT", "MX"}, "description": "record type (default A)"}, + "server": map[string]any{"type": "string", "description": "ask this resolver alone: an address, with :port or without (53), e.g. 10.10.0.1"}, + "tcp": map[string]any{"type": "boolean", "description": "ask the server over TCP rather than UDP (default false)"}, "timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 3000, at most 30000)"}, }, Run: func(args map[string]any) (any, error) { @@ -65,7 +93,9 @@ func tools() []stdio.Tool { if err != nil { return nil, err } - return CheckDNS(name, kind, timeout) + server, _ := args["server"].(string) + overTCP, _ := args["tcp"].(bool) + return AskDNS(name, kind, strings.TrimSpace(server), overTCP, timeout) }, }, } diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json index 9ee8e8f5..c8f87901 100644 --- a/modules/netcheck/module.json +++ b/modules/netcheck/module.json @@ -4,8 +4,20 @@ "tools": [ "netcheck_tcp", "netcheck_dns", - "netcheck_http" + "netcheck_http", + "netcheck_listening" ], + "replaces": { + "netcheck_listening": [ + "ss -lunt", + "ss -ltn", + "netstat -lnt" + ], + "netcheck_dns": [ + "dig", + "nslookup" + ] + }, "build": { "artifacts": [ { From 12de28c81d74c57f6b661a0414d33cfa52519e8c Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:33:23 +0200 Subject: [PATCH 4/6] docker: read the resolver file a container really uses (hq issue 348) On 2026-10-09 the control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the machine's file listed both mesh resolvers; its copy lives under the runtime's root-only directory and no tool read it. docker_resolv_conf reads /etc/resolv.conf as the container sees it through the runtime's archive (docker cp, no exec), parses it, and says whether it lists the same resolvers as the machine. --- modules/docker/cmd/docker-tools/main.go | 13 ++ modules/docker/cmd/docker-tools/resolv.go | 111 ++++++++++++++++++ .../docker/cmd/docker-tools/resolv_test.go | 83 +++++++++++++ modules/docker/module.json | 6 + 4 files changed, 213 insertions(+) create mode 100644 modules/docker/cmd/docker-tools/resolv.go create mode 100644 modules/docker/cmd/docker-tools/resolv_test.go diff --git a/modules/docker/cmd/docker-tools/main.go b/modules/docker/cmd/docker-tools/main.go index 9e95a39d..66abc0bf 100644 --- a/modules/docker/cmd/docker-tools/main.go +++ b/modules/docker/cmd/docker-tools/main.go @@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool { return c.Inspect(ctx, ref) }, }, + { + Name: "docker_resolv_conf", + Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " + + "its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.", + Input: map[string]any{"container": containerArg}, + Run: func(args map[string]any) (any, error) { + ref, err := text(args, "container") + if err != nil { + return nil, err + } + return c.ResolvConfOf(ctx, ref) + }, + }, { Name: "docker_logs", Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " + diff --git a/modules/docker/cmd/docker-tools/resolv.go b/modules/docker/cmd/docker-tools/resolv.go new file mode 100644 index 00000000..e04750fa --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv.go @@ -0,0 +1,111 @@ +package main + +// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's +// resolver file, made by the runtime when it is created and kept under the runtime's root-only +// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the +// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the +// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the +// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream): +// no exec into the container, nothing written, and it works where the program has no shell. + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "strings" +) + +// ContainerResolvConf is the path read inside a container. +const ContainerResolvConf = "/etc/resolv.conf" + +// HostResolvConf is the machine's own resolver file, read for comparison. +const HostResolvConf = "/etc/resolv.conf" + +// ResolvConf is a resolver file, parsed. +type ResolvConf struct { + Nameservers []string `json:"nameservers"` + Search []string `json:"search,omitempty"` + Options []string `json:"options,omitempty"` + Raw string `json:"raw"` +} + +// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only. +func parseResolvConf(text string) ResolvConf { + out := ResolvConf{Nameservers: []string{}, Raw: text} + for _, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") { + continue + } + switch f[0] { + case "nameserver": + if len(f) > 1 { + out.Nameservers = append(out.Nameservers, f[1]) + } + case "search", "domain": + out.Search = append(out.Search, f[1:]...) + case "options": + out.Options = append(out.Options, f[1:]...) + } + } + return out +} + +// fileFromArchive is the one regular file a `docker cp : -` stream carries. +func fileFromArchive(stream string) (string, error) { + tr := tar.NewReader(strings.NewReader(stream)) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return "", errors.New("the runtime's archive held no file") + } + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + switch h.Typeflag { + case tar.TypeReg: + body, err := io.ReadAll(io.LimitReader(tr, 64<<10)) + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + return string(body), nil + case tar.TypeSymlink: + return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname) + } + } +} + +// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list +// the same resolvers in the same order. +func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) { + ref, err := Ref(ref) + if err != nil { + return nil, err + } + answer := map[string]any{"container": ref, "path": ContainerResolvConf} + stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-") + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + text, err := fileFromArchive(stream) + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + inside := parseResolvConf(text) + answer["resolv_conf"] = inside + if raw, err := c.ReadFile(HostResolvConf); err != nil { + answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error() + } else { + host := parseResolvConf(string(raw)) + answer["machine_nameservers"] = host.Nameservers + same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ") + answer["same_as_machine"] = same + if !same { + answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " + + "or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again" + } + } + return answer, nil +} diff --git a/modules/docker/cmd/docker-tools/resolv_test.go b/modules/docker/cmd/docker-tools/resolv_test.go new file mode 100644 index 00000000..7512c7c3 --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func archiveOf(t *testing.T, h tar.Header, body string) string { + t.Helper() + var buf bytes.Buffer + w := tar.NewWriter(&buf) + h.Size = int64(len(body)) + if h.Typeflag == tar.TypeSymlink { + h.Size = 0 + } + if err := w.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if h.Typeflag != tar.TypeSymlink { + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + w.Close() + return buf.String() +} + +// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy +// of the file could not be read by anything on the mesh. +func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) { + inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n" + f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -", + Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)}) + c := client(f, 1000) + c.ReadFile = func(p string) ([]byte, error) { + if p != HostResolvConf { + t.Fatalf("read %s", p) + } + return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil + } + got, err := c.ResolvConfOf(context.Background(), "route-proxy") + if err != nil { + t.Fatal(err) + } + rc := got["resolv_conf"].(ResolvConf) + if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" || + strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside { + t.Fatalf("parsed: %+v", rc) + } + if got["same_as_machine"] != false || got["note"] == nil { + t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got) + } + if f.ran("docker exec") { + t.Fatal("read by exec, not through the archive") + } +} + +func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) { + c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Fatalf("a missing container: %v", err) + } + link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "") + c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") { + t.Fatalf("a link inside: %v", err) + } + c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil { + t.Fatal("garbage was read as a file") + } + ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n") + c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000) + c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") } + got, err := c.ResolvConfOf(context.Background(), "x") + if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil { + t.Fatalf("an unreadable machine file: %v, %v", got, err) + } +} diff --git a/modules/docker/module.json b/modules/docker/module.json index b7fc0726..73c8f74c 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -18,6 +18,7 @@ "tools": [ "docker_list", "docker_inspect", + "docker_resolv_conf", "docker_logs", "docker_secrets_in_logs", "docker_secrets_in_events", @@ -38,6 +39,11 @@ "docker_problems", "docker_ports" ], + "replaces": { + "docker_resolv_conf": [ + "docker exec cat /etc/resolv.conf" + ] + }, "resources": [ { "id": "package", From a0049f3e4ee0383c66bb52b857f79a2a647197de Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 13:50:20 +0200 Subject: [PATCH 5/6] dnsmasq: bind each address once, at start, after it exists (hq issue 348) bind-dynamic closes every listener whose interface a re-read of the machine's addresses did not find, and a re-read during a bridge teardown fails (dnsmasq 2.93 network.c:832-875, reproduced in a network namespace: 53 releases in 120 parallel teardowns). bind-interfaces never closes a listener; a drop-in makes the unit wait for the private address and retry a start that found it missing. --- modules/dnsmasq/module.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 4469a297..9974ead7 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -69,7 +69,20 @@ "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n#\n# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the\n# `mesh-dns-resolver` seat, which more than one machine may hold, each answering\n# the same names from the same roster. Every node and every container lists every\n# holder and no public resolver, so whichever answers first gives the one answer.\n# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the\n# mesh's, rendered into each holder; no other copy lives on any machine.\n\n# What the mesh computed: one wildcard per machine — its name and everything\n# under it — and the mesh's own suffix as a local domain, so a name under it is\n# answered here or not at all and is never asked upstream. Rewritten whenever a\n# machine joins or leaves, which is why the service below restarts on it: a\n# reload makes dnsmasq re-read hosts files, not its configuration, and a\n# wildcard is configuration.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to\n# the private address of the machine its module runs on and the port its\n# answerer is published on. Nothing in a zone is answered here; what names a zone\n# holds is the module's. Rewritten whenever a zone is declared or moves, and the\n# service restarts on it for the same reason as above.\nconf-file=/etc/mesh-resolver/zones.conf\n\n# Where it answers: this machine's private address, so every node — and every\n# container on every node, which copies its machine's resolvers — can ask; and\n# loopback, for this machine's own use. Never a LAN address: a device that is\n# not a member cannot reach what the mesh's names point at, and a LAN's resolver\n# is its router's (ADR 0194).\n#\n# Named as an ADDRESS and not as the interface that carries it, on purpose. A\n# container's query is addressed to this address but arrives on the runtime's\n# bridge, and dnsmasq checks every query against what it was told to answer on:\n# an `interface=` line admits queries by the interface they arrive on, a\n# `listen-address=` line by the address they are sent to. With `interface=mesh0`\n# alone, a query from a container was received and dropped without a word\n# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on.\n#\n# bind-dynamic rather than bind-interfaces: the private address does not exist\n# until the machine is on the private network, and binding an address that is\n# not there yet fails to start rather than waiting for it.\nbind-dynamic\nlisten-address=${machine:address}\nlisten-address=127.0.0.1\n\n# **It must never read resolv.conf to find out where to forward.** This\n# machine's resolv.conf names this resolver — so a resolver that read it for\n# upstreams would find itself, and every query it could not answer locally would\n# loop until its receive queue filled. That is not theoretical: it filled with\n# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop\n# impossible: the upstreams are the two lines below, and nothing on the machine\n# can redirect them. The mesh's own names never reach them: the local= line in\n# the file above stops them here, answered or refused.\nno-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n\n# Both upstreams validate DNSSEC and say so with the Authenticated Data bit,\n# and this passes that bit down rather than dropping it, which dnsmasq does\n# unless told. It does not validate itself: an answer's trust is the upstream's\n# and the path to it, which is what a forwarding resolver's trust always was.\n# Without it a program that refuses to run behind a resolver that does not\n# validate — a mail system's admin does exactly that check at start — cannot\n# use this resolver (novox/hq 04-ISSUES/171).\nproxy-dnssec\n\n# A name without a dot is never forwarded, and reverse lookups of private ranges\n# are answered here rather than asking the world who 10.x is.\ndomain-needed\nbogus-priv\n\n# **No hosts file, and no operator's files.** This machine's /etc/hosts is its\n# own — the hosts module's block and the operator's lines (ADR 0199) — and the\n# mesh's resolver answers every node, so a line written for one machine's own\n# programs must not become an answer for all of them. Every per-node resolver\n# went wrong exactly here: a hosts file read once at start, an operator's old\n# line beside the mesh's, a drop-in read from a directory nobody owned.\nno-hosts\n" + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n#\n# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the\n# `mesh-dns-resolver` seat, which more than one machine may hold, each answering\n# the same names from the same roster. Every node and every container lists every\n# holder and no public resolver, so whichever answers first gives the one answer.\n# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the\n# mesh's, rendered into each holder; no other copy lives on any machine.\n\n# What the mesh computed: one wildcard per machine — its name and everything\n# under it — and the mesh's own suffix as a local domain, so a name under it is\n# answered here or not at all and is never asked upstream. Rewritten whenever a\n# machine joins or leaves, which is why the service below restarts on it: a\n# reload makes dnsmasq re-read hosts files, not its configuration, and a\n# wildcard is configuration.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to\n# the private address of the machine its module runs on and the port its\n# answerer is published on. Nothing in a zone is answered here; what names a zone\n# holds is the module's. Rewritten whenever a zone is declared or moves, and the\n# service restarts on it for the same reason as above.\nconf-file=/etc/mesh-resolver/zones.conf\n\n# Where it answers: this machine's private address, so every node — and every\n# container on every node, which copies its machine's resolvers — can ask; and\n# loopback, for this machine's own use. Never a LAN address: a device that is\n# not a member cannot reach what the mesh's names point at, and a LAN's resolver\n# is its router's (ADR 0194).\n#\n# Named as an ADDRESS and not as the interface that carries it, on purpose. A\n# container's query is addressed to this address but arrives on the runtime's\n# bridge, and dnsmasq checks every query against what it was told to answer on:\n# an `interface=` line admits queries by the interface they arrive on, a\n# `listen-address=` line by the address they are sent to. With `interface=mesh0`\n# alone, a query from a container was received and dropped without a word\n# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on.\n#\n# bind-interfaces, and never bind-dynamic (novox/hq issue 348). bind-dynamic\n# re-reads the machine's addresses on every change of any link and closes each\n# listener whose interface that reading did not find; a reading made while the\n# container runtime deletes a bridge's virtual links fails (\"index gone\"), and\n# then every listener is closed until a later change reads cleanly. On\n# 2026-10-09 the private address was left unbound for twenty minutes, the daemon\n# running and silent. bind-interfaces binds each address once, at start, and\n# never closes it. It cannot bind an address that is not there yet, so the unit\n# waits for it (the drop-in below); a new private address restarts the unit, as\n# this file changing always did.\nbind-interfaces\nlisten-address=${machine:address}\nlisten-address=127.0.0.1\n\n# **It must never read resolv.conf to find out where to forward.** This\n# machine's resolv.conf names this resolver — so a resolver that read it for\n# upstreams would find itself, and every query it could not answer locally would\n# loop until its receive queue filled. That is not theoretical: it filled with\n# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop\n# impossible: the upstreams are the two lines below, and nothing on the machine\n# can redirect them. The mesh's own names never reach them: the local= line in\n# the file above stops them here, answered or refused.\nno-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n\n# Both upstreams validate DNSSEC and say so with the Authenticated Data bit,\n# and this passes that bit down rather than dropping it, which dnsmasq does\n# unless told. It does not validate itself: an answer's trust is the upstream's\n# and the path to it, which is what a forwarding resolver's trust always was.\n# Without it a program that refuses to run behind a resolver that does not\n# validate — a mail system's admin does exactly that check at start — cannot\n# use this resolver (novox/hq 04-ISSUES/171).\nproxy-dnssec\n\n# A name without a dot is never forwarded, and reverse lookups of private ranges\n# are answered here rather than asking the world who 10.x is.\ndomain-needed\nbogus-priv\n\n# **No hosts file, and no operator's files.** This machine's /etc/hosts is its\n# own — the hosts module's block and the operator's lines (ADR 0199) — and the\n# mesh's resolver answers every node, so a line written for one machine's own\n# programs must not become an answer for all of them. Every per-node resolver\n# went wrong exactly here: a hosts file read once at start, an operator's old\n# line beside the mesh's, a drop-in read from a directory nobody owned.\nno-hosts\n" + }, + { + "id": "drop-ins", + "type": "directory", + "path": "/etc/systemd/system/dnsmasq.service.d", + "mode": "0755" + }, + { + "id": "wait-for-address", + "type": "file", + "path": "/etc/systemd/system/dnsmasq.service.d/mesh.conf", + "mode": "0644", + "content": "# Written by the mesh (module dnsmasq, novox/hq issue 348). Replaced on every push.\n#\n# The resolver binds its addresses once, at start (bind-interfaces), so it starts only once this\n# machine's private address is there: it waits up to 80 seconds for it, and a start that found it\n# missing is tried again every ten seconds rather than left failed.\n[Unit]\nAfter=network-online.target\nWants=network-online.target\nStartLimitIntervalSec=0\n\n[Service]\nExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 80); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo \"${machine:address} is not on this machine\" >&2; exit 1'\nRestart=on-failure\nRestartSec=10s\n" }, { "id": "service", @@ -79,6 +92,7 @@ "boot": "enabled", "restart-on": [ "config", + "wait-for-address", "dnsmasq.fact-node-zones", "dnsmasq.fact-zones" ], From a6a9a818d2c81074cdb69efecdde8cbe16c464bb Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 9 Oct 2026 15:04:43 +0200 Subject: [PATCH 6/6] Answer the review of hq issue 348's catalogue change - The dnsmasq watch restarts only a unit that runs, without waiting for its start; a unit stopped, starting or failed is held and said (C2). - dnsmasq_resolve asks only an address the resolver listens on (C3). - Tests for the watch loop keeping its budget, a watch that wrote nothing or stopped looking, and a unit not running (C4). - A restart is remembered as said only once emitted, restarts made before the bundle started are said, and whether one helped is said (C5). - docker_events reads the runtime's whole buffer: complete when it is not full or reaches past the window, never a quiet machine read as lost (C6). - netcheck: refused read with errors.Is; tests for addresses and ports, and for a container named as docker knows it. --- .gitignore | 2 + .../dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go | 251 ++++++++++++++++-- modules/dnsmasq/cmd/dnsmasq-tools/main.go | 117 ++++---- modules/dnsmasq/cmd/dnsmasq-tools/tools.go | 22 +- modules/dnsmasq/cmd/dnsmasq-tools/watch.go | 36 ++- modules/dnsmasq/module.json | 3 +- modules/docker/cmd/docker-tools/docker.go | 55 ++-- .../docker/cmd/docker-tools/docker_test.go | 40 ++- .../docker/cmd/docker-tools/resolv_test.go | 13 + .../docker/cmd/docker-tools/secrets_test.go | 3 +- modules/netcheck/cmd/netcheck/ask_test.go | 12 + modules/netcheck/cmd/netcheck/checks.go | 3 +- 12 files changed, 437 insertions(+), 120 deletions(-) diff --git a/.gitignore b/.gitignore index 01c8bb0b..36dbb3ca 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools modules/messenger/cmd/messenger/messenger modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher modules/claude-code/cmd/claude-code/claude-code +modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools # ...and the same built from the module's root (go build ./cmd/), which names it after the module. modules/messenger/messenger modules/mesh-watcher/mesh-watcher +modules/dnsmasq/dnsmasq diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go index 197f0ed4..c9ae3b9a 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/binary" + "errors" + "fmt" "net" "os" "path/filepath" @@ -11,6 +13,8 @@ import ( "sync" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until @@ -205,15 +209,15 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) var s State var why string - if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 { + if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 { t.Fatalf("one failing look restarted: %q", why) } - if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { + if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { t.Fatalf("a second failing look did not restart: %+v %q", s, why) } s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why}) // It helped. - s, why = Step(s, healthy(t0.Add(2*LookEvery))) + s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active") if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped { t.Fatalf("a healthy look after a restart: %+v %q", s, why) } @@ -222,7 +226,7 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { restarts := 1 for at.Add(LookEvery).Sub(t0) < 50*time.Minute { at = at.Add(LookEvery) - s, why = Step(s, failing(at)) + s, why = Step(s, failing(at), "active") if why == "" { continue } @@ -247,8 +251,8 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { // A look that could not ask — no configuration — restarts nothing. var none State broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}} - none, _ = Step(none, broken) - if _, why := Step(none, broken); why != "" { + none, _ = Step(none, broken, "active") + if _, why := Step(none, broken, "active"); why != "" { t.Fatalf("a missing configuration restarted the resolver: %q", why) } } @@ -281,7 +285,7 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp var s State look := func() { t.Helper() - next, why := Step(s, m.Look(t.Context())) + next, why := Step(s, m.Look(t.Context()), "active") if why != "" { r := Restart{At: now().UTC(), Why: why} if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil { @@ -330,23 +334,226 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp } } -func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) { - var a Announcer - t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) - old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}} - if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 { - t.Fatalf("said at its start: %+v", got) +// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came +// back on its own, or by something else. +func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + s := State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active") + if h := s.Restarts[0].Helped; h == nil || *h { + t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h) } - if got := a.Next(nil, false, old); len(got) != 0 { - t.Fatalf("an unreadable roster said %+v", got) + s = State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active") + if h := s.Restarts[0].Helped; h == nil || !*h { + t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h) } - now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})} - got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now) - types := map[string]int{} - for _, e := range got { - types[e.Type]++ +} + +// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is +// held, said, and not counted against the budget (review of PR 161, C2). +func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + for _, unit := range []string{"inactive", "activating", "failed", ""} { + var s State + var why string + for i := 0; i < 4; i++ { + s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit) + if why != "" { + t.Fatalf("a unit %q was restarted", unit) + } + } + if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 { + t.Fatalf("a unit %q: %+v", unit, s) + } + // Running again and still not answering: restarted, the held looks counted as failing. + if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" { + t.Fatalf("a unit running again after %q, not answering, was not restarted", unit) + } } - if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 { - t.Fatalf("said %+v", got) +} + +// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts +// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4). +func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer { + return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"} + } + before := LookEvery + LookEvery = time.Millisecond + t.Cleanup(func() { LookEvery = before }) + run := func(spent []Restart) ([]string, State) { + t.Helper() + if err := m.WriteState(State{Restarts: spent}); err != nil { + t.Fatal(err) + } + var asked []string + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + looks := 0 + var last State + err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if args[0] == "show" { + return "active\n", nil + } + return "", nil + }, func(s State) error { + last = s + if looks++; looks == 3 { + cancel() + } + return m.WriteState(s) + }) + if err != nil { + t.Fatal(err) + } + return asked, last + } + // Three restarts within the hour before it was restarted: it restarts nothing more. + spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}} + asked, last := run(spent) + for _, a := range asked { + if strings.Contains(a, "restart") { + t.Fatalf("restarted with its budget spent before it started: %v", asked) + } + } + if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 { + t.Fatalf("the budget it kept: %+v", last) + } + // None spent: restarted on its second failing look, not waiting for the start. + asked, last = run(nil) + restarts := 0 + for _, a := range asked { + if strings.HasPrefix(a, "systemctl restart") { + restarts++ + if a != "systemctl restart --no-block dnsmasq.service" { + t.Fatalf("restarted as %q", a) + } + } + } + if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" { + t.Fatalf("restarts %d: %v %+v", restarts, asked, last) + } +} + +// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running. +func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") { + t.Fatalf("no watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") { + t.Fatalf("a stale watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !h.Healthy || h.Why != "" { + t.Fatalf("a watch looking, a resolver answering: %+v", h) + } + if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") { + t.Fatalf("a unit stopped: %+v", h.Why) + } + if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") { + t.Fatalf("an unreadable state: %+v", h.Why) + } +} + +// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3). +func TestResolveAsksOnlyAListenAddress(t *testing.T) { + port := freePort(t) + m := aResolverMachine(t, port, time.Now) + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") { + t.Fatalf("another address was asked: %v", err) + } + got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp") + if err != nil { + t.Fatal(err) + } + answers := got.(map[string]any)["answers"].([]Answer) + if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" { + t.Fatalf("%+v", answers) + } + if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 { + t.Fatalf("every listen address over both: %+v", got) + } + if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil { + t.Fatal("over quic was accepted") + } + if _, err := m.Resolve(t.Context(), "", "", ""); err == nil { + t.Fatal("no name was accepted") + } +} + +// The announcer: names primed silently, restarts made before the bundle started said, an event remembered +// only once emitted, and whether a restart helped said (review of PR 161, C5). +func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) { + port := freePort(t) + t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return t0 }) + yes := true + if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil { + t.Fatal(err) + } + var said []string + failing := true + emit := func(kind string, body any, _ ...stdio.EmitOptions) error { + if failing && kind == "restarted" { + return errors.New("no bus") + } + said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"])) + return nil + } + var a Announcer + Tell(m, &a, emit) + if strings.Join(said, "|") != "restart-judged true" { + t.Fatalf("at the start: %v", said) + } + failing, said = false, nil + Tell(m, &a, emit) + if strings.Join(said, "|") != "restarted " { + t.Fatalf("a restart whose emit failed was not said again: %v", said) + } + said = nil + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("said twice: %v", said) + } + // The roster changes: one machine added, one gone. An unreadable roster says nothing. + nodes := filepath.Join(m.Root, NodesFile) + if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") || + !strings.Contains(got, "name.removed ace.internal") || len(said) != 2 { + t.Fatalf("the roster's change: %v", said) + } + said = nil + if err := os.Remove(nodes); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("an unreadable roster said %v", said) } } diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/main.go b/modules/dnsmasq/cmd/dnsmasq-tools/main.go index 771d3ee5..91f33cc2 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/main.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/main.go @@ -45,84 +45,101 @@ func main() { // AnnounceEvery is how often the bundle reads the roster and the watch for what to say. var AnnounceEvery = 30 * time.Second -// Announcer remembers what was said, primed silently so a restart of the bundle does not say every name -// and every restart again. +// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not +// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when +// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded. type Announcer struct { - names map[string]string - restarts map[time.Time]bool - // primed says the names were read once; heard that the watch's restarts were. - primed, heard bool + names map[string]string + primed bool + // said is every restart said, by when it was made; judged every one whose outcome was said. + said, judged map[time.Time]bool } -// Event is one event to emit. +// Event is one event to emit, and what saying it records. type Event struct { Type string Body map[string]any + said func() } -// Next is what changed since the last read: names the resolver now answers (`name.added`) or no longer -// does (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own -// events — and restarts the watch made (`restarted`), each with why and whether it helped once known. -// -// read says the roster was read: an unreadable roster says nothing about names, rather than every name -// removed. +// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does +// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events; +// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it +// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names, +// rather than every name removed. An event is remembered by Said, once it was emitted. func (a *Announcer) Next(names []Named, read bool, s State) []Event { - now := map[string]string{} - for _, n := range names { - now[n.Name] = n.Address + if a.said == nil { + a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{} } var out []Event - if a.primed && read { - for name, address := range now { - if _, had := a.names[name]; !had { - out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}}) + if read { + now := map[string]string{} + for _, n := range names { + now[n.Name] = n.Address + } + if !a.primed { + a.names, a.primed = now, true + } else { + for name, address := range now { + if _, had := a.names[name]; !had { + out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}, + func() { a.names[name] = address }}) + } + } + for name := range a.names { + if _, has := now[name]; !has { + out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }}) + } } } - for name := range a.names { - if _, has := now[name]; !has { - out = append(out, Event{"name.removed", map[string]any{"name": name}}) - } - } - } - if a.restarts == nil { - a.restarts = map[time.Time]bool{} } for _, r := range s.Restarts { - if a.restarts[r.At] { - continue - } - a.restarts[r.At] = true - if a.heard { - body := map[string]any{"unit": Unit, "at": r.At, "why": r.Why} + at := r.At + if !a.said[at] { + body := map[string]any{"unit": Unit, "at": at, "why": r.Why} if r.Error != "" { body["error"] = r.Error } - out = append(out, Event{"restarted", body}) + out = append(out, Event{"restarted", body, func() { a.said[at] = true }}) + } + if r.Helped != nil && !a.judged[at] { + out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped}, + func() { a.judged[at] = true }}) } } - a.heard = true - if read { - a.names, a.primed = now, true - } return out } +// Said remembers an event as said. +func (e Event) Said() { + if e.said != nil { + e.said() + } +} + // Announce says the events for as long as the bundle runs. func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) { time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs var a Announcer for { - var names []Named - text, err := m.read(NodesFile) - if err == nil { - names = AnsweredNames(text) - } - s, _, _ := m.ReadState() - for _, e := range a.Next(names, err == nil, s) { - if err := emit(e.Type, e.Body); err != nil { - fmt.Fprintf(os.Stderr, "dnsmasq: %s not said: %v\n", e.Type, err) - } - } + Tell(m, &a, emit) time.Sleep(AnnounceEvery) } } + +// Tell is one read and what it says: an event that could not be emitted is said again at the next read. +func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) { + var names []Named + text, err := m.read(NodesFile) + if err == nil { + names = AnsweredNames(text) + } + s, _, _ := m.ReadState() + for _, e := range a.Next(names, err == nil, s) { + if err := emit(e.Type, e.Body); err != nil { + fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err) + continue + } + e.Said() + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go index 3d4d11e2..8d5e5cde 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "slices" "strings" "time" @@ -48,6 +49,9 @@ func (m *Machine) Health(ctx context.Context) Health { if !l.Healthy { why = append(why, restartWords(s)) } + if s.Unit != "" && s.Unit != "active" { + why = append(why, "the unit is "+s.Unit) + } } h.Why = strings.Join(nonEmpty(why), "; ") return h @@ -56,7 +60,7 @@ func (m *Machine) Health(ctx context.Context) Health { // restartWords is what the watch did about it, in words. func restartWords(s State) string { if len(s.Restarts) == 0 { - return "the watch has not restarted it" + return "the watch has not restarted it" + orEmpty("; ", s.Held) } r := s.Restarts[len(s.Restarts)-1] words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC")) @@ -132,15 +136,17 @@ func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, if name == "" { return nil, fmt.Errorf("name is required") } - var addresses []string + config, err := m.read(ConfigFile) + if err != nil { + return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + } + addresses := ListenAddresses(config) if address != "" { - addresses = []string{address} - } else { - config, err := m.read(ConfigFile) - if err != nil { - return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + if !slices.Contains(addresses, address) { + return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver", + address, strings.Join(addresses, ", ")) } - addresses = ListenAddresses(config) + addresses = []string{address} } overs := []string{"udp", "tcp"} switch over { diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go index 4989dc8c..fed79191 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go @@ -64,8 +64,11 @@ type State struct { Since *time.Time `json:"since,omitempty"` // Restarts are the restarts within the last hour and the last one ever, newest last. Restarts []Restart `json:"restarts,omitempty"` - // Held is why a restart that was due was not made: the budget is spent. + // Held is why a restart that was due was not made: the budget is spent, or the unit is not running. Held string `json:"held,omitempty"` + // Unit is the resolver unit's state at the last look, as the service manager said it: active, + // activating, inactive, failed … + Unit string `json:"unit,omitempty"` } // Runner runs one command; injected, so the watch is tested over recorded answers. @@ -83,10 +86,13 @@ func ExecRunner(ctx context.Context, name string, args ...string) (string, error } // Step is one look of the watch and what follows from it: the state after, and whether to restart and why. -// Pure but for the look it is given. -func Step(s State, l Look) (State, string) { +// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of +// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at +// boot for the private address — or failed is the service manager's, and none of them is a resolver that +// runs and does not answer. Pure. +func Step(s State, l Look, unit string) (State, string) { now := l.At - s.Last = l + s.Last, s.Unit = l, unit // What the earlier restarts are now known to have done. for i := range s.Restarts { r := &s.Restarts[i] @@ -124,6 +130,11 @@ func Step(s State, l Look) (State, string) { // configuration or roster is missing. return s, "" } + if unit != "active" { + s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+ + "answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit)) + return s, "" + } within := 0 var newest *Restart for i, r := range s.Restarts { @@ -155,10 +166,16 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error defer tick.Stop() for { l := m.Look(ctx) - next, why := Step(s, l) + unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit) + if err != nil { + unit = "" + } + next, why := Step(s, l, strings.TrimSpace(unit)) if why != "" { r := Restart{At: m.Now().UTC(), Why: why} - if out, err := run(ctx, "systemctl", "restart", Unit); err != nil { + // Not waited for: a start that waits for the private address is the service manager's to time, + // and the next looks say whether it answers. + if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil { r.Error = strings.TrimSpace(err.Error() + " " + out) } fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error)) @@ -178,6 +195,13 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error } } +func orUnknown(s string) string { + if s == "" { + return "not known (the service manager did not say)" + } + return s +} + func orEmpty(prefix, s string) string { if s == "" { return "" diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 9974ead7..d868d771 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -18,7 +18,8 @@ "emits": [ "name.added", "name.removed", - "restarted" + "restarted", + "restart-judged" ], "own-secrets": { "broker": "${dir:mesh-state}/broker" diff --git a/modules/docker/cmd/docker-tools/docker.go b/modules/docker/cmd/docker-tools/docker.go index 7411ac21..88669917 100644 --- a/modules/docker/cmd/docker-tools/docker.go +++ b/modules/docker/cmd/docker-tools/docker.go @@ -1011,15 +1011,24 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind) } } - args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"} + // The whole of what the runtime holds, from the epoch: its buffer, which says where its history really + // begins and whether it is full; the window is cut from it here. + args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"} out, err := c.docker(ctx, args...) if err != nil { return nil, err } - raw, err := jsonLines[runtimeEvent](out) + all, err := jsonLines[runtimeEvent](out) if err != nil { return nil, err } + start := now.Add(-time.Duration(minutes) * time.Minute) + raw := make([]runtimeEvent, 0, len(all)) + for _, e := range all { + if !time.Unix(0, e.TimeNano).Before(start) { + raw = append(raw, e) + } + } // An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer // is read by agents and kept in their transcripts, which would make it a second copy of the leak. envs := &envCache{c: c, ctx: ctx} @@ -1057,41 +1066,47 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind events = events[len(events)-limit:] } answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events, - "held": len(raw)} + "held": len(all)} if len(redacted) > 0 { answer["redacted"] = sortedSet(redacted) answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)" } - for k, v := range historyReach(raw, now, minutes) { + for k, v := range historyReach(all, start) { answer[k] = v } return answer, nil } -// historyGap is how much later than the window's start the runtime's oldest held event may be and the -// window still read as covered: events are not made every second. -const historyGap = 2 * time.Minute +// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer +// holding fewer has every event since the runtime started; a full one has lost what came before its oldest. +const RuntimeEventsKept = 256 -// historyReach says where the runtime's history begins when it begins after the window asked for: the -// oldest event it still holds, and that nothing before it can be answered from here. -func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any { - start := now.Add(-time.Duration(minutes) * time.Minute) - if len(raw) == 0 { - return map[string]any{"complete": false, - "history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"} +// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window +// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not +// full — then it holds everything since the runtime started. Otherwise where the history begins, and that +// nothing before it can be answered from here. +func historyReach(all []runtimeEvent, start time.Time) map[string]any { + if len(all) == 0 { + return map[string]any{"complete": true, "oldest_held": nil, + "history": "the runtime holds no events at all: none happened since it started"} } - oldest := time.Unix(0, raw[0].TimeNano) - for _, e := range raw { + oldest := time.Unix(0, all[0].TimeNano) + for _, e := range all { if t := time.Unix(0, e.TimeNano); t.Before(oldest) { oldest = t } } - if oldest.Sub(start) <= historyGap { - return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)} + at := oldest.UTC().Format(time.RFC3339) + if len(all) < RuntimeEventsKept { + return map[string]any{"complete": true, "oldest_held": at, + "history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)} } - return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339), + if !oldest.After(start) { + return map[string]any{"complete": true, "oldest_held": at} + } + return map[string]any{"complete": false, "oldest_held": at, "history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)", - len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} + len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} } func sortedSet(set map[string]bool) []string { diff --git a/modules/docker/cmd/docker-tools/docker_test.go b/modules/docker/cmd/docker-tools/docker_test.go index 41ff8eb7..7bf8042f 100644 --- a/modules/docker/cmd/docker-tools/docker_test.go +++ b/modules/docker/cmd/docker-tools/docker_test.go @@ -300,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { {"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2} ` f := (&fake{}).on("docker events", Ran{Stdout: out}) - got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false) if err != nil { t.Fatal(err) } @@ -309,7 +309,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { t.Fatalf("%v", evs) } // The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348). - if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") { + if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { t.Fatalf("%+v", f.calls) } if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil { @@ -376,15 +376,15 @@ func eventLine(at time.Time, kind, action, name string) string { func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) { now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) var out strings.Builder - for i := 0; i < 250; i++ { + for i := 0; i < RuntimeEventsKept; i++ { out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail")) } got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false) if err != nil { t.Fatal(err) } - if got["count"] != 0 || got["held"] != 250 { - t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"]) + if got["count"] != 0 || got["held"] != RuntimeEventsKept { + t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept) } if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") || got["oldest_held"] != "2026-10-09T11:14:00Z" { @@ -394,7 +394,10 @@ func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *te func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) { now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) - out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + + // A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer + // is not full, so it holds every event since the runtime started. + out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") + + eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") + eventLine(now.Add(-1*time.Minute), "network", "create", "build-net") f := (&fake{}).on("docker events", Ran{Stdout: out}) @@ -402,14 +405,29 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test if err != nil { t.Fatal(err) } - if got["complete"] != true || got["count"] != 2 || got["history"] != nil { - t.Fatalf("a covered window: %v", got) + if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") { + t.Fatalf("a quiet machine's window: %v", got) } - if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") { + if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { t.Fatal("the type was filtered by the runtime, so the history's start is unknown") } + // A quiet machine asked for ten minutes, its newest event an hour old: complete, and none. + got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("a quiet machine read as one whose history was lost: %v", got) + } + // No events at all: none since the runtime started. got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) - if got["complete"] != false || got["history"] == nil { - t.Fatalf("no events at all reads as quiet: %v", got) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("an empty buffer: %v", got) + } + // A full buffer whose oldest is before the window: complete. + var full strings.Builder + for i := 0; i < RuntimeEventsKept; i++ { + full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x")) + } + got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) + if got["complete"] != true || got["history"] != nil { + t.Fatalf("a full buffer reaching past the window: %v", got) } } diff --git a/modules/docker/cmd/docker-tools/resolv_test.go b/modules/docker/cmd/docker-tools/resolv_test.go index 7512c7c3..6c1285e8 100644 --- a/modules/docker/cmd/docker-tools/resolv_test.go +++ b/modules/docker/cmd/docker-tools/resolv_test.go @@ -81,3 +81,16 @@ func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) { t.Fatalf("an unreadable machine file: %v, %v", got, err) } } + +// The container is named as docker knows it, never as something docker would read as an option. +func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) { + f := &fake{} + for _, ref := range []string{"--privileged", "a b", "", "x;rm"} { + if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil { + t.Errorf("%q was asked of the runtime", ref) + } + } + if len(f.calls) != 0 { + t.Fatalf("the runtime was asked: %+v", f.calls) + } +} diff --git a/modules/docker/cmd/docker-tools/secrets_test.go b/modules/docker/cmd/docker-tools/secrets_test.go index ef2fa257..bd8baaf8 100644 --- a/modules/docker/cmd/docker-tools/secrets_test.go +++ b/modules/docker/cmd/docker-tools/secrets_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "strings" "testing" + "time" ) // The shape of the leak in hq issue 268: a server password announced at start and a database URI @@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) { f := (&fake{}). on("docker events", Ran{Stdout: execEvents}). on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"}) - got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true) if err != nil { t.Fatal(err) } diff --git a/modules/netcheck/cmd/netcheck/ask_test.go b/modules/netcheck/cmd/netcheck/ask_test.go index 19ca738a..1ad2c293 100644 --- a/modules/netcheck/cmd/netcheck/ask_test.go +++ b/modules/netcheck/cmd/netcheck/ask_test.go @@ -115,6 +115,18 @@ func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) { if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" { t.Fatalf("default port: %s", a) } + for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53", + "[::1]:53": "[::1]:53", "::1": "[::1]:53"} { + if a, err := serverAddress(server); err != nil || a != want { + t.Errorf("%q: %q, %v; want %q", server, a, err, want) + } + } + for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536", + "10.10.0.1:dns", ""} { + if a, err := serverAddress(server); err == nil { + t.Errorf("%q was taken as %q", server, a) + } + } } func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) { diff --git a/modules/netcheck/cmd/netcheck/checks.go b/modules/netcheck/cmd/netcheck/checks.go index 8fbf74a9..db7ca08c 100644 --- a/modules/netcheck/cmd/netcheck/checks.go +++ b/modules/netcheck/cmd/netcheck/checks.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "syscall" "time" ) @@ -220,7 +221,7 @@ func timeoutOf(args map[string]any) (time.Duration, error) { func dnsWords(err error, timeout time.Duration) string { var ne net.Error switch { - case strings.Contains(err.Error(), "connection refused"): + case errors.Is(err, syscall.ECONNREFUSED): return "connection refused: nothing listens on that address and port" case errors.As(err, &ne) && ne.Timeout(): return fmt.Sprintf("no answer within %s", timeout)