diff --git a/.gitignore b/.gitignore index dfd78dd4..1ba033d5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,11 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools modules/messenger/cmd/messenger/messenger modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher modules/claude-code/cmd/claude-code/claude-code +modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery # ...and the same built from the module's root (go build ./cmd/), which names it after the module. modules/messenger/messenger modules/claude-code/claude-code modules/mesh-watcher/mesh-watcher +modules/dnsmasq/dnsmasq modules/mesh-delivery/mesh-delivery diff --git a/health-undeclared b/health-undeclared index c69be7a1..4165b15c 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1,4 +1,4 @@ -70 +69 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running diff --git a/modules/dnsmasq/client.ts b/modules/dnsmasq/client.ts deleted file mode 100644 index ed6a5be1..00000000 --- a/modules/dnsmasq/client.ts +++ /dev/null @@ -1,62 +0,0 @@ -// dnsmasq's own code, living in the module (novox/hq ADR 0039). dnsmasq here answers the mesh's -// generated wildcard names (..) itself and forwards everything else to fixed -// upstreams — the predecessor's arrangement, so the machine and its containers ask it for the world. -// Its code reads what it was told to answer, and can resolve through itself to prove that it does. - -import { readFile } from "node:fs/promises"; -import { Resolver } from "node:dns/promises"; - -export interface AnsweredName { - /** A machine's internal name, e.g. "anchor.internal" — it and everything under it resolve here. */ - name: string; - address: string; -} - -export class DnsmasqClient { - constructor( - private readonly resolverPath: string, - private readonly address: string, - ) {} - - /** - * Build from the environment. Both values are node-local facts with mesh-chosen defaults — the - * wildcard file the module is sent, and the loopback address its config listens on (127.0.0.1, - * where the predecessor's resolver answered and every machine's resolv.conf already points) — so - * there is nothing to be unconfigured about; it never throws. - */ - static fromEnv(env: NodeJS.ProcessEnv = process.env): DnsmasqClient { - return new DnsmasqClient( - env.MESH_DNSMASQ_RESOLVER_PATH ?? "/etc/mesh-resolver/nodes.conf", - env.MESH_DNSMASQ_ADDRESS ?? "127.0.0.1", - ); - } - - /** The names this resolver answers, read from the mesh-generated wildcard file. */ - async answeredNames(): Promise { - let text: string; - try { - text = await readFile(this.resolverPath, "utf8"); - } catch { - return []; // not yet on the network, or the file has not been written — no names, not an error - } - const names: AnsweredName[] = []; - for (const line of text.split("\n")) { - // dnsmasq wildcard syntax the mesh writes: address=/./
. The file also - // carries local=//, which names no machine and is not matched here. - const match = line.match(/^address=\/([^/]+)\/(.+)$/); - if (match) names.push({ name: match[1], address: match[2] }); - } - return names; - } - - /** Resolve a name through this node's own resolver — the check that wildcard-resolution answers. */ - async resolve(name: string): Promise { - const resolver = new Resolver(); - resolver.setServers([this.address]); - try { - return await resolver.resolve4(name); - } catch { - return await resolver.resolve6(name); - } - } -} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/ask.go b/modules/dnsmasq/cmd/dnsmasq-tools/ask.go new file mode 100644 index 00000000..9bcc4c5a --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/ask.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math/rand/v2" + "net" + "strings" + "syscall" + "time" +) + +// A real question to the resolver, as any program on the machine asks it: one name, one record type, over +// UDP or TCP, to one address. Written out here rather than taken from a library: the module needs the +// header's answer code and the A records, nothing else, and Go's own resolver would read /etc/hosts and +// /etc/resolv.conf instead of asking the one address it is told (novox/hq issue 348). + +// Answer is what one question got. +type Answer struct { + // Server is where it was asked, and Over the transport. + Server string `json:"server"` + Over string `json:"over"` + Name string `json:"name"` + // Answered says a reply came; Code is its answer code (NOERROR, NXDOMAIN, …) and Addresses its A + // records. + Answered bool `json:"answered"` + Code string `json:"code,omitempty"` + Addresses []string `json:"addresses,omitempty"` + // Failed is why no reply came, in words: refused (nothing listens there), no answer within, or what + // the system said. + Failed string `json:"failed,omitempty"` + // Took is how long it took, in milliseconds. + Took int64 `json:"took_ms"` +} + +// rcodes are the answer codes by number. +var rcodes = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"} + +// query is one question for name's A record, with id. +func query(name string, id uint16) ([]byte, error) { + name = strings.TrimSuffix(name, ".") + if name == "" { + return nil, errors.New("no name to ask") + } + msg := make([]byte, 12, 64) + binary.BigEndian.PutUint16(msg[0:], id) + binary.BigEndian.PutUint16(msg[2:], 0x0100) // a standard query, recursion desired + binary.BigEndian.PutUint16(msg[4:], 1) // one question + for _, label := range strings.Split(name, ".") { + if label == "" || len(label) > 63 { + return nil, fmt.Errorf("%q is not a name that can be asked", name) + } + msg = append(msg, byte(len(label))) + msg = append(msg, label...) + } + msg = append(msg, 0, 0, 1, 0, 1) // root; type A; class IN + return msg, nil +} + +// readAnswer reads a reply to the question with id: its code and A records. +func readAnswer(msg []byte, id uint16) (string, []string, error) { + if len(msg) < 12 { + return "", nil, errors.New("a reply too short to be one") + } + if binary.BigEndian.Uint16(msg[0:]) != id { + return "", nil, errors.New("a reply to another question") + } + flags := binary.BigEndian.Uint16(msg[2:]) + code, ok := rcodes[int(flags&0xF)] + if !ok { + code = fmt.Sprintf("RCODE%d", flags&0xF) + } + qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:])) + at := 12 + for i := 0; i < qd; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return code, nil, err + } + at += 4 + } + var addresses []string + for i := 0; i < an; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return code, addresses, err + } + if at+10 > len(msg) { + return code, addresses, errors.New("a reply cut short") + } + kind, length := binary.BigEndian.Uint16(msg[at:]), int(binary.BigEndian.Uint16(msg[at+8:])) + at += 10 + if at+length > len(msg) { + return code, addresses, errors.New("a reply cut short") + } + if kind == 1 && length == 4 { + addresses = append(addresses, net.IP(msg[at:at+4]).String()) + } + at += length + } + return code, addresses, nil +} + +// skipName steps over a name in a message, a pointer ending it. +func skipName(msg []byte, at int) (int, error) { + for at < len(msg) { + n := int(msg[at]) + switch { + case n == 0: + return at + 1, nil + case n&0xC0 == 0xC0: + return at + 2, nil + } + at += n + 1 + } + return at, errors.New("a name runs past the reply") +} + +// Ask asks server (an address, port 53 unless given) for name's A record over "udp" or "tcp", within +// timeout. +func Ask(ctx context.Context, server, over, name string, timeout time.Duration) Answer { + if _, _, err := net.SplitHostPort(server); err != nil { + server = net.JoinHostPort(server, "53") + } + a := Answer{Server: server, Over: over, Name: name} + started := time.Now() + defer func() { a.Took = time.Since(started).Milliseconds() }() + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + id := uint16(rand.UintN(1 << 16)) + q, err := query(name, id) + if err != nil { + a.Failed = err.Error() + return a + } + conn, err := (&net.Dialer{}).DialContext(ctx, over, server) + if err != nil { + a.Failed = failedWords(err, timeout) + a.Took = time.Since(started).Milliseconds() + return a + } + defer conn.Close() + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + var reply []byte + if over == "tcp" { + framed := binary.BigEndian.AppendUint16(nil, uint16(len(q))) + if _, err = conn.Write(append(framed, q...)); err == nil { + head := make([]byte, 2) + if _, err = readFull(conn, head); err == nil { + reply = make([]byte, binary.BigEndian.Uint16(head)) + _, err = readFull(conn, reply) + } + } + } else if _, err = conn.Write(q); err == nil { + buf := make([]byte, 1500) + var n int + n, err = conn.Read(buf) + reply = buf[:n] + } + if err != nil { + a.Failed = failedWords(err, timeout) + a.Took = time.Since(started).Milliseconds() + return a + } + a.Answered = true + a.Code, a.Addresses, err = readAnswer(reply, id) + if err != nil { + a.Failed = err.Error() + } + a.Took = time.Since(started).Milliseconds() + return a +} + +func readFull(conn net.Conn, buf []byte) (int, error) { + read := 0 + for read < len(buf) { + n, err := conn.Read(buf[read:]) + read += n + if err != nil { + return read, err + } + } + return read, nil +} + +// failedWords is why no reply came: refused when nothing listens, no answer when nothing came in time. +func failedWords(err error, timeout time.Duration) string { + var timedOut interface{ Timeout() bool } + switch { + case errors.Is(err, syscall.ECONNREFUSED): + return "refused: nothing listens there" + case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timedOut) && timedOut.Timeout()): + return fmt.Sprintf("no answer within %s", timeout) + } + return err.Error() +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go new file mode 100644 index 00000000..c9ae3b9a --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go @@ -0,0 +1,559 @@ +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until +// stopped — as dnsmasq answers the machine's own name from the roster. +type standIn struct { + udp net.PacketConn + tcp net.Listener + wg sync.WaitGroup +} + +func reply(q []byte, address string) []byte { + // The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name. + out := append([]byte{}, q...) + binary.BigEndian.PutUint16(out[2:], 0x8180) + binary.BigEndian.PutUint16(out[6:], 1) + ip := net.ParseIP(address).To4() + out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4) + return append(out, ip...) +} + +func startStandIn(t *testing.T, at, port, address string) *standIn { + t.Helper() + s := &standIn{} + var err error + if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil { + t.Fatal(err) + } + if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil { + t.Fatal(err) + } + s.wg.Add(2) + go func() { + defer s.wg.Done() + buf := make([]byte, 1500) + for { + n, from, err := s.udp.ReadFrom(buf) + if err != nil { + return + } + _, _ = s.udp.WriteTo(reply(buf[:n], address), from) + } + }() + go func() { + defer s.wg.Done() + for { + c, err := s.tcp.Accept() + if err != nil { + return + } + head := make([]byte, 2) + if _, err := readFull(c, head); err == nil { + q := make([]byte, binary.BigEndian.Uint16(head)) + if _, err := readFull(c, q); err == nil { + r := reply(q, address) + _, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...)) + } + } + _ = c.Close() + } + }() + t.Cleanup(s.stop) + return s +} + +func (s *standIn) stop() { + _ = s.udp.Close() + _ = s.tcp.Close() + s.wg.Wait() +} + +// freePort is a port free on both loopback addresses the tests use. +func freePort(t *testing.T) string { + t.Helper() + for i := 0; i < 20; i++ { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + _ = l.Close() + ok := true + for _, at := range []string{"127.0.0.1", "127.0.0.2"} { + for _, over := range []string{"tcp", "udp"} { + if over == "tcp" { + if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil { + _ = c.Close() + } else { + ok = false + } + } else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil { + _ = c.Close() + } else { + ok = false + } + } + } + if ok { + return port + } + } + t.Fatal("no port free on both loopback addresses") + return "" +} + +// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address) +// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2. +func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine { + t.Helper() + root := t.TempDir() + write := func(path, text string) { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n") + write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+ + "address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n") + if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil { + t.Fatal(err) + } + return &Machine{Root: root, Ask: Ask, Now: now, Port: port} +} + +func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) { + listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n") + if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" { + t.Fatalf("listen addresses %v", listen) + } + names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" + + "host-record=novox.internal,10.77.0.1\n") + q, ok := Question(listen, names) + if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" { + t.Fatalf("asks %+v", q) + } + if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" { + t.Fatalf("with no own name in the roster, asks %+v", q) + } +} + +// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback, +// and refused every question on its private address. Its unit said healthy. The look does not. +func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) { + port := freePort(t) + m := aResolverMachine(t, port, time.Now) + private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" { + t.Fatalf("a resolver answering everywhere: %+v", l) + } + private.stop() + l := m.Look(t.Context()) + if l.Healthy || len(l.Wrong) != 2 { + t.Fatalf("a resolver that stopped answering on its private address: %+v", l) + } + if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") || + !strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") { + t.Fatalf("said as %q", why) + } +} + +func TestAWrongAnswerIsNotAnAnswer(t *testing.T) { + q := Named{Name: "novox.internal", Address: "10.77.0.1"} + for _, c := range []struct { + a Answer + want string + }{ + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""}, + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"}, + {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"}, + {Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"}, + } { + if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v said %q, want %q", c.a, got, c.want) + } + } +} + +func failing(at time.Time) Look { + return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}}, + Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}} +} + +func healthy(at time.Time) Look { return Look{At: at, Healthy: true} } + +// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks +// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why. +func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + var s State + var why string + if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 { + t.Fatalf("one failing look restarted: %q", why) + } + if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { + t.Fatalf("a second failing look did not restart: %+v %q", s, why) + } + s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why}) + // It helped. + s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active") + if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped { + t.Fatalf("a healthy look after a restart: %+v %q", s, why) + } + // Failing again soon after: held for the gap, then restarted; and so on until the budget is spent. + at := t0.Add(3 * LookEvery) + restarts := 1 + for at.Add(LookEvery).Sub(t0) < 50*time.Minute { + at = at.Add(LookEvery) + s, why = Step(s, failing(at), "active") + if why == "" { + continue + } + last := s.Restarts[len(s.Restarts)-1].At + if at.Sub(last) < RestartGap { + t.Fatalf("restarted %s after the last", at.Sub(last)) + } + s.Restarts = append(s.Restarts, Restart{At: at, Why: why}) + restarts++ + } + if restarts != RestartsPerHour { + t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour) + } + if !strings.Contains(s.Held, "the most the watch makes") { + t.Fatalf("the spent budget is not said: %+v", s) + } + for _, r := range s.Restarts[1:] { + if r.Helped == nil || *r.Helped { + t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped) + } + } + // A look that could not ask — no configuration — restarts nothing. + var none State + broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}} + none, _ = Step(none, broken, "active") + if _, why := Step(none, broken, "active"); why != "" { + t.Fatalf("a missing configuration restarted the resolver: %q", why) + } +} + +// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the +// watch notices on its second look, restarts the unit (which brings the listener back), says the restart +// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not +// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the +// operator. +func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + var mu sync.Mutex + now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock } + pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() } + m := aResolverMachine(t, port, now) + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") + private.stop() + + fixes := true + var asked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if fixes { + private = startStandIn(t, "127.0.0.2", port, "127.0.0.2") + } + return "", nil + } + var s State + look := func() { + t.Helper() + next, why := Step(s, m.Look(t.Context()), "active") + if why != "" { + r := Restart{At: now().UTC(), Why: why} + if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil { + r.Error = err.Error() + } + next.Restarts = append(next.Restarts, r) + } + s = next + if err := m.WriteState(s); err != nil { + t.Fatal(err) + } + pass(LookEvery) + } + look() + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") { + t.Fatalf("after one failing look: %+v", h.Why) + } + look() + if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" { + t.Fatalf("restarted with %v", asked) + } + look() + if r := s.Restarts[0]; r.Helped == nil || !*r.Helped { + t.Fatalf("the restart that brought it back is said as %+v", r) + } + if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 { + t.Fatalf("after the restart helped: %+v", h) + } + + // It stops again, and this time a restart does not help. + private.stop() + fixes = false + pass(RestartGap) + for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ { + look() + } + h := m.Health(t.Context()) + if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") || + !strings.Contains(h.Why, "which did not help") { + t.Fatalf("a restart that did not help: %q", h.Why) + } + // And a watch that stopped looking is said, whatever the resolver does. + pass(Stale + time.Minute) + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") { + t.Fatalf("a stale watch: %q", h.Why) + } +} + +// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came +// back on its own, or by something else. +func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + s := State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active") + if h := s.Restarts[0].Helped; h == nil || *h { + t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h) + } + s = State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active") + if h := s.Restarts[0].Helped; h == nil || !*h { + t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h) + } +} + +// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is +// held, said, and not counted against the budget (review of PR 161, C2). +func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + for _, unit := range []string{"inactive", "activating", "failed", ""} { + var s State + var why string + for i := 0; i < 4; i++ { + s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit) + if why != "" { + t.Fatalf("a unit %q was restarted", unit) + } + } + if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 { + t.Fatalf("a unit %q: %+v", unit, s) + } + // Running again and still not answering: restarted, the held looks counted as failing. + if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" { + t.Fatalf("a unit running again after %q, not answering, was not restarted", unit) + } + } +} + +// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts +// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4). +func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer { + return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"} + } + before := LookEvery + LookEvery = time.Millisecond + t.Cleanup(func() { LookEvery = before }) + run := func(spent []Restart) ([]string, State) { + t.Helper() + if err := m.WriteState(State{Restarts: spent}); err != nil { + t.Fatal(err) + } + var asked []string + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + looks := 0 + var last State + err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if args[0] == "show" { + return "active\n", nil + } + return "", nil + }, func(s State) error { + last = s + if looks++; looks == 3 { + cancel() + } + return m.WriteState(s) + }) + if err != nil { + t.Fatal(err) + } + return asked, last + } + // Three restarts within the hour before it was restarted: it restarts nothing more. + spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}} + asked, last := run(spent) + for _, a := range asked { + if strings.Contains(a, "restart") { + t.Fatalf("restarted with its budget spent before it started: %v", asked) + } + } + if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 { + t.Fatalf("the budget it kept: %+v", last) + } + // None spent: restarted on its second failing look, not waiting for the start. + asked, last = run(nil) + restarts := 0 + for _, a := range asked { + if strings.HasPrefix(a, "systemctl restart") { + restarts++ + if a != "systemctl restart --no-block dnsmasq.service" { + t.Fatalf("restarted as %q", a) + } + } + } + if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" { + t.Fatalf("restarts %d: %v %+v", restarts, asked, last) + } +} + +// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running. +func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") { + t.Fatalf("no watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") { + t.Fatalf("a stale watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !h.Healthy || h.Why != "" { + t.Fatalf("a watch looking, a resolver answering: %+v", h) + } + if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") { + t.Fatalf("a unit stopped: %+v", h.Why) + } + if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") { + t.Fatalf("an unreadable state: %+v", h.Why) + } +} + +// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3). +func TestResolveAsksOnlyAListenAddress(t *testing.T) { + port := freePort(t) + m := aResolverMachine(t, port, time.Now) + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") { + t.Fatalf("another address was asked: %v", err) + } + got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp") + if err != nil { + t.Fatal(err) + } + answers := got.(map[string]any)["answers"].([]Answer) + if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" { + t.Fatalf("%+v", answers) + } + if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 { + t.Fatalf("every listen address over both: %+v", got) + } + if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil { + t.Fatal("over quic was accepted") + } + if _, err := m.Resolve(t.Context(), "", "", ""); err == nil { + t.Fatal("no name was accepted") + } +} + +// The announcer: names primed silently, restarts made before the bundle started said, an event remembered +// only once emitted, and whether a restart helped said (review of PR 161, C5). +func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) { + port := freePort(t) + t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return t0 }) + yes := true + if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil { + t.Fatal(err) + } + var said []string + failing := true + emit := func(kind string, body any, _ ...stdio.EmitOptions) error { + if failing && kind == "restarted" { + return errors.New("no bus") + } + said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"])) + return nil + } + var a Announcer + Tell(m, &a, emit) + if strings.Join(said, "|") != "restart-judged true" { + t.Fatalf("at the start: %v", said) + } + failing, said = false, nil + Tell(m, &a, emit) + if strings.Join(said, "|") != "restarted " { + t.Fatalf("a restart whose emit failed was not said again: %v", said) + } + said = nil + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("said twice: %v", said) + } + // The roster changes: one machine added, one gone. An unreadable roster says nothing. + nodes := filepath.Join(m.Root, NodesFile) + if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") || + !strings.Contains(got, "name.removed ace.internal") || len(said) != 2 { + t.Fatalf("the roster's change: %v", said) + } + said = nil + if err := os.Remove(nodes); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("an unreadable roster said %v", said) + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/look.go b/modules/dnsmasq/cmd/dnsmasq-tools/look.go new file mode 100644 index 00000000..996664aa --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/look.go @@ -0,0 +1,207 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// A look asks the resolver one real question on every address its configuration tells it to answer on, +// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat +// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its +// journal silent, and refused every question on its private address for twenty minutes; its unit said +// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers. + +// The files a look reads, under the machine's root. +const ( + ConfigFile = "/etc/dnsmasq.conf" + NodesFile = "/etc/mesh-resolver/nodes.conf" +) + +// AskWithin bounds each question. +const AskWithin = 2 * time.Second + +// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock. +type Machine struct { + Root string + Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer + Now func() time.Time + // Port is where the resolver answers; 53 but in a test. + Port string +} + +// Here is this machine. +func Here() *Machine { + return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"} +} + +func (m *Machine) read(path string) (string, error) { + b, err := os.ReadFile(filepath.Join(m.Root, path)) + return string(b), err +} + +// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order. +func ListenAddresses(config string) []string { + var out []string + sc := bufio.NewScanner(strings.NewReader(config)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + value, ok := strings.CutPrefix(line, "listen-address=") + if !ok { + continue + } + for _, a := range strings.Split(value, ",") { + if a = strings.TrimSpace(a); a != "" { + out = append(out, a) + } + } + } + return out +} + +// Named is a machine's name and its private address, as the roster gave them to the resolver. +type Named struct { + Name string `json:"name"` + Address string `json:"address"` +} + +// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address: +// its `host-record=,
` lines. +func AnsweredNames(nodes string) []Named { + var out []Named + sc := bufio.NewScanner(strings.NewReader(nodes)) + for sc.Scan() { + value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=") + if !ok { + continue + } + name, address, ok := strings.Cut(value, ",") + if ok && name != "" && address != "" { + out = append(out, Named{Name: name, Address: strings.TrimSpace(address)}) + } + } + return out +} + +// Question is the name a look asks, and the address a right answer carries: the machine's own when the +// roster names an address the resolver listens on, else the roster's first. +func Question(listen []string, names []Named) (Named, bool) { + for _, n := range names { + for _, a := range listen { + if n.Address == a { + return n, true + } + } + } + if len(names) > 0 { + return names[0], true + } + return Named{}, false +} + +// Look is one look: every listen address asked over both transports. +type Look struct { + At time.Time `json:"at"` + // Healthy says every question was answered rightly. + Healthy bool `json:"healthy"` + Asked Named `json:"asked"` + Answers []Answer `json:"answers"` + // Wrong is every question not answered rightly, in words; Problems what kept the look from asking. + Wrong []string `json:"wrong,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// Why is what is wrong in one sentence: empty when healthy. +func (l Look) Why() string { + if l.Healthy { + return "" + } + return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ") +} + +// Look asks every listen address its question, over UDP and TCP, all at once. +func (m *Machine) Look(ctx context.Context) Look { + l := Look{At: m.Now().UTC()} + config, err := m.read(ConfigFile) + if err != nil { + l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err)) + return l + } + listen := ListenAddresses(config) + if len(listen) == 0 { + l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile)) + return l + } + nodes, err := m.read(NodesFile) + if err != nil { + l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err)) + return l + } + q, ok := Question(listen, AnsweredNames(nodes)) + if !ok { + l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile)) + return l + } + l.Asked = q + var mu sync.Mutex + var wg sync.WaitGroup + for _, address := range listen { + for _, over := range []string{"udp", "tcp"} { + wg.Add(1) + go func(address, over string) { + defer wg.Done() + a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin) + mu.Lock() + l.Answers = append(l.Answers, a) + mu.Unlock() + }(address, over) + } + } + wg.Wait() + sort.Slice(l.Answers, func(i, j int) bool { + if l.Answers[i].Server != l.Answers[j].Server { + return l.Answers[i].Server < l.Answers[j].Server + } + return l.Answers[i].Over > l.Answers[j].Over + }) + for _, a := range l.Answers { + if w := wrongWords(a, q); w != "" { + l.Wrong = append(l.Wrong, w) + } + } + l.Healthy = len(l.Wrong) == 0 + return l +} + +// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the +// machine's address. +func wrongWords(a Answer, q Named) string { + at := a.Server + " over " + a.Over + switch { + case !a.Answered: + return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed) + case a.Code != "NOERROR": + return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code) + case a.Failed != "": + return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed) + } + for _, x := range a.Addresses { + if x == q.Address { + return "" + } + } + return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address) +} + +func orNothing(s []string) string { + if len(s) == 0 { + return "no address" + } + return strings.Join(s, ", ") +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/main.go b/modules/dnsmasq/cmd/dnsmasq-tools/main.go new file mode 100644 index 00000000..91f33cc2 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/main.go @@ -0,0 +1,145 @@ +// The dnsmasq module's Go bundle: one of the mesh's resolvers (novox/hq ADR 0194, 0223). +// +// Started with no arguments it is served by the node's runtime as the operator account, over MCP on stdio +// (ADR 0188, ADR 0193): the module's tools and its health, and beside them the events the resolver's +// answers give — a machine's name it now answers or no longer does, and a restart the watch made. Started +// as `dnsmasq-tools watch` it is the module's long-running process, as root: it asks the resolver a real +// question on every listen address and restarts it, within a budget, when it stops answering (watch.go, +// novox/hq issue 348). stdout is the MCP channel; everything else is said on stderr. +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func main() { + if len(os.Args) > 1 { + if os.Args[1] != "watch" || len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: dnsmasq-tools [watch]") + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) + defer stop() + m := Here() + if err := m.Watch(ctx, ExecRunner, m.WriteState); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + m := Here() + go Announce(m, stdio.Emit) + if err := stdio.Serve("", Tools(m)); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// AnnounceEvery is how often the bundle reads the roster and the watch for what to say. +var AnnounceEvery = 30 * time.Second + +// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not +// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when +// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded. +type Announcer struct { + names map[string]string + primed bool + // said is every restart said, by when it was made; judged every one whose outcome was said. + said, judged map[time.Time]bool +} + +// Event is one event to emit, and what saying it records. +type Event struct { + Type string + Body map[string]any + said func() +} + +// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does +// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events; +// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it +// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names, +// rather than every name removed. An event is remembered by Said, once it was emitted. +func (a *Announcer) Next(names []Named, read bool, s State) []Event { + if a.said == nil { + a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{} + } + var out []Event + if read { + now := map[string]string{} + for _, n := range names { + now[n.Name] = n.Address + } + if !a.primed { + a.names, a.primed = now, true + } else { + for name, address := range now { + if _, had := a.names[name]; !had { + out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}, + func() { a.names[name] = address }}) + } + } + for name := range a.names { + if _, has := now[name]; !has { + out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }}) + } + } + } + } + for _, r := range s.Restarts { + at := r.At + if !a.said[at] { + body := map[string]any{"unit": Unit, "at": at, "why": r.Why} + if r.Error != "" { + body["error"] = r.Error + } + out = append(out, Event{"restarted", body, func() { a.said[at] = true }}) + } + if r.Helped != nil && !a.judged[at] { + out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped}, + func() { a.judged[at] = true }}) + } + } + return out +} + +// Said remembers an event as said. +func (e Event) Said() { + if e.said != nil { + e.said() + } +} + +// Announce says the events for as long as the bundle runs. +func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) { + time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs + var a Announcer + for { + Tell(m, &a, emit) + time.Sleep(AnnounceEvery) + } +} + +// Tell is one read and what it says: an event that could not be emitted is said again at the next read. +func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) { + var names []Named + text, err := m.read(NodesFile) + if err == nil { + names = AnsweredNames(text) + } + s, _, _ := m.ReadState() + for _, e := range a.Next(names, err == nil, s) { + if err := emit(e.Type, e.Body); err != nil { + fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err) + continue + } + e.Said() + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go new file mode 100644 index 00000000..8d5e5cde --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Stale is how old the watch's last look may be before the health tool says it stopped looking. +const Stale = 2 * time.Minute + +// Health is the module's health answer (novox/hq ADR 0240): healthy only when the resolver answers its +// real question on every listen address, over UDP and TCP, and the watch is looking. +type Health struct { + Healthy bool `json:"healthy"` + Why string `json:"why,omitempty"` + Look Look `json:"look"` + // Watch is what the watch last saw and did; absent when it has written nothing yet. + Watch *State `json:"watch,omitempty"` +} + +// Health asks the resolver now, and reads what the watch did. +func (m *Machine) Health(ctx context.Context) Health { + l := m.Look(ctx) + h := Health{Healthy: l.Healthy, Look: l} + var why []string + if !l.Healthy { + why = append(why, "the resolver does not answer as it must: "+l.Why()) + } + s, had, err := m.ReadState() + switch { + case err != nil: + h.Healthy = false + why = append(why, "what the watch saw cannot be read: "+err.Error()) + case !had: + h.Healthy = false + why = append(why, "the watch has written nothing yet, so nothing restarts the resolver when it stops answering") + default: + h.Watch = &s + if age := m.Now().Sub(s.Last.At); age > Stale { + h.Healthy = false + why = append(why, fmt.Sprintf("the watch last looked %s ago, so nothing restarts the resolver when it "+ + "stops answering", age.Round(time.Second))) + } + if !l.Healthy { + why = append(why, restartWords(s)) + } + if s.Unit != "" && s.Unit != "active" { + why = append(why, "the unit is "+s.Unit) + } + } + h.Why = strings.Join(nonEmpty(why), "; ") + return h +} + +// restartWords is what the watch did about it, in words. +func restartWords(s State) string { + if len(s.Restarts) == 0 { + return "the watch has not restarted it" + orEmpty("; ", s.Held) + } + r := s.Restarts[len(s.Restarts)-1] + words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC")) + switch { + case r.Error != "": + words += ", and the restart failed: " + r.Error + case r.Helped != nil && !*r.Helped: + words += ", which did not help" + case r.Helped == nil: + words += ", and its next looks decide whether that helped" + } + if s.Held != "" { + words += "; " + s.Held + } + return words +} + +func nonEmpty(s []string) []string { + var out []string + for _, x := range s { + if x != "" { + out = append(out, x) + } + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func arg(a map[string]any, k string) string { + v, _ := a[k].(string) + return strings.TrimSpace(v) +} + +// Tools are the module's tools. +func Tools(m *Machine) []stdio.Tool { + return []stdio.Tool{ + {Name: "dnsmasq_health", Description: "Whether this machine's mesh resolver answers: its real question — this " + + "machine's own name — asked now on every address its configuration listens on, over UDP and TCP, each " + + "answer with its code, addresses and time; and what the module's watch last saw, the restarts it made " + + "within the hour, whether each helped, and why it holds back from another. Healthy only when every " + + "question was answered rightly and the watch is looking (novox/hq ADR 0240, issue 348). (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { + return m.Health(context.Background()), nil + }}, + {Name: "dnsmasq_names", Description: "The mesh's machine names this machine's resolver answers, each with its " + + "private address, as the roster wrote them (" + NodesFile + "). (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { + text, err := m.read(NodesFile) + if err != nil { + return nil, fmt.Errorf("the roster %s cannot be read: %w", NodesFile, err) + } + return map[string]any{"names": AnsweredNames(text)}, nil + }}, + {Name: "dnsmasq_resolve", Description: "Ask this machine's resolver one name's IPv4 address, directly, on one " + + "address it listens on (default every one) over udp or tcp (default both): the answer code, the addresses " + + "and the time each took, or why none came — refused when nothing listens there. Replaces dig @address. (r)", + Input: map[string]any{"name": str("the name to ask, e.g. plex.anchor.internal"), + "address": str("one listen address to ask (default: every one in its configuration)"), + "over": str("udp or tcp (default: both)")}, + Run: func(a map[string]any) (any, error) { + return m.Resolve(context.Background(), arg(a, "name"), arg(a, "address"), arg(a, "over")) + }}, + } +} + +// Resolve asks name on one listen address or every one, over one transport or both. +func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, error) { + if name == "" { + return nil, fmt.Errorf("name is required") + } + config, err := m.read(ConfigFile) + if err != nil { + return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + } + addresses := ListenAddresses(config) + if address != "" { + if !slices.Contains(addresses, address) { + return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver", + address, strings.Join(addresses, ", ")) + } + addresses = []string{address} + } + overs := []string{"udp", "tcp"} + switch over { + case "": + case "udp", "tcp": + overs = []string{over} + default: + return nil, fmt.Errorf("over is udp or tcp, not %q", over) + } + var answers []Answer + for _, a := range addresses { + for _, o := range overs { + answers = append(answers, m.Ask(ctx, a+":"+m.Port, o, name, AskWithin)) + } + } + return map[string]any{"name": name, "answers": answers}, nil +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go new file mode 100644 index 00000000..fed79191 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go @@ -0,0 +1,240 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real +// question on every listen address (look.go); after Confirm failing looks in a row it restarts the +// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh +// (novox/hq issue 348). +// +// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that +// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor +// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where +// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked, +// and it never says it worked: its next look does. A restart that did not help is said by the health +// tool as the resolver's failure, which the node-engine states and the controller raises — the operator +// hears of it only then. + +// The watch's clocks and its budget. +var ( + LookEvery = 15 * time.Second + // Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277). + Confirm = 2 + // RestartGap is the least time between two restarts; RestartsPerHour the most within an hour. + RestartGap = 2 * time.Minute + RestartsPerHour = 3 + // HelpedWithin is how soon after a restart a healthy look says the restart helped. + HelpedWithin = time.Minute +) + +// Unit is the resolver's service. +const Unit = "dnsmasq.service" + +// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator +// account the tools run as, written only by the watch. +const StateFile = "/var/lib/dnsmasq-watch/watch.json" + +// Restart is one restart the watch made. +type Restart struct { + At time.Time `json:"at"` + Why string `json:"why"` + // Error is what the service manager said when the restart itself failed. + Error string `json:"error,omitempty"` + // Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once + // HelpedWithin passed without one, absent until then. + Helped *bool `json:"helped,omitempty"` +} + +// State is the watch's last look and what it did, as the tools read it. +type State struct { + Last Look `json:"last"` + // Failing is how many looks in a row failed; Since when the first of them looked. + Failing int `json:"failing"` + Since *time.Time `json:"since,omitempty"` + // Restarts are the restarts within the last hour and the last one ever, newest last. + Restarts []Restart `json:"restarts,omitempty"` + // Held is why a restart that was due was not made: the budget is spent, or the unit is not running. + Held string `json:"held,omitempty"` + // Unit is the resolver unit's state at the last look, as the service manager said it: active, + // activating, inactive, failed … + Unit string `json:"unit,omitempty"` +} + +// Runner runs one command; injected, so the watch is tested over recorded answers. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// ExecRunner runs a command on the machine, bounded to a minute. +func ExecRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + var out bytes.Buffer + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return strings.TrimSpace(out.String()), err +} + +// Step is one look of the watch and what follows from it: the state after, and whether to restart and why. +// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of +// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at +// boot for the private address — or failed is the service manager's, and none of them is a resolver that +// runs and does not answer. Pure. +func Step(s State, l Look, unit string) (State, string) { + now := l.At + s.Last, s.Unit = l, unit + // What the earlier restarts are now known to have done. + for i := range s.Restarts { + r := &s.Restarts[i] + if r.Helped != nil || r.Error != "" { + continue + } + switch { + case l.Healthy && now.Sub(r.At) <= HelpedWithin: + yes := true + r.Helped = &yes + case now.Sub(r.At) > HelpedWithin: + no := false + r.Helped = &no + } + } + // The budget's window: the last hour, and always the newest. + kept := s.Restarts[:0] + for i, r := range s.Restarts { + if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 { + kept = append(kept, r) + } + } + s.Restarts = kept + if l.Healthy { + s.Failing, s.Since, s.Held = 0, nil, "" + return s, "" + } + if s.Failing == 0 { + at := now + s.Since = &at + } + s.Failing++ + if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 { + // Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing + // configuration or roster is missing. + return s, "" + } + if unit != "active" { + s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+ + "answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit)) + return s, "" + } + within := 0 + var newest *Restart + for i, r := range s.Restarts { + if now.Sub(r.At) <= time.Hour { + within++ + } + newest = &s.Restarts[i] + } + switch { + case newest != nil && now.Sub(newest.At) < RestartGap: + s.Held = "" + return s, "" // the last restart is still settling + case within >= RestartsPerHour: + s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+ + "more until the oldest is an hour old, and the resolver's failure is said as its health", within) + return s, "" + } + s.Held = "" + return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why()) +} + +// Watch runs the watch until ctx ends. +func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error { + var s State + if raw, err := m.read(StateFile); err == nil { + _ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart + } + tick := time.NewTicker(LookEvery) + defer tick.Stop() + for { + l := m.Look(ctx) + unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit) + if err != nil { + unit = "" + } + next, why := Step(s, l, strings.TrimSpace(unit)) + if why != "" { + r := Restart{At: m.Now().UTC(), Why: why} + // Not waited for: a start that waits for the private address is the service manager's to time, + // and the next looks say whether it answers. + if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil { + r.Error = strings.TrimSpace(err.Error() + " " + out) + } + fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error)) + next.Restarts = append(next.Restarts, r) + } else if !l.Healthy && next.Failing == Confirm { + fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held)) + } + s = next + if err := write(s); err != nil { + fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err) + } + select { + case <-ctx.Done(): + return nil + case <-tick.C: + } + } +} + +func orUnknown(s string) string { + if s == "" { + return "not known (the service manager did not say)" + } + return s +} + +func orEmpty(prefix, s string) string { + if s == "" { + return "" + } + return prefix + s +} + +// WriteState writes the state whole, replacing the file in one rename. +func (m *Machine) WriteState(s State) error { + path := filepath.Join(m.Root, StateFile) + body, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + tmp := path + ".new" + if err := os.WriteFile(tmp, body, 0o644); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// ReadState is the watch's last state, and false when it has written none. +func (m *Machine) ReadState() (State, bool, error) { + raw, err := m.read(StateFile) + if errors.Is(err, os.ErrNotExist) { + return State{}, false, nil + } + if err != nil { + return State{}, false, err + } + var s State + if err := json.Unmarshal([]byte(raw), &s); err != nil { + return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err) + } + return s, true, nil +} diff --git a/modules/dnsmasq/go.mod b/modules/dnsmasq/go.mod new file mode 100644 index 00000000..d439da63 --- /dev/null +++ b/modules/dnsmasq/go.mod @@ -0,0 +1,5 @@ +module dnsmasq + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.7 diff --git a/modules/dnsmasq/go.sum b/modules/dnsmasq/go.sum new file mode 100644 index 00000000..b474419e --- /dev/null +++ b/modules/dnsmasq/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= +git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/dnsmasq/index.ts b/modules/dnsmasq/index.ts deleted file mode 100644 index d8327bdb..00000000 --- a/modules/dnsmasq/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -// dnsmasq's events. The resolver's answered set changes whenever the mesh rewrites the wildcard file -// and restarts it — a machine joined the private network or left it. This watches that file and -// announces, from the resolver's own vantage, that a name became (or stopped being) resolvable on -// this node: DNS having actually propagated here, distinct from the mesh's own node.* events. -// -// Emits (novox/hq ADR 0041/0042): -// module.dnsmasq.name.added — this node's resolver now answers a machine's name -// module.dnsmasq.name.removed — it no longer does - -import { emit } from "@novox/mesh-sdk/events"; -import { DnsmasqClient } from "./client.js"; - -const dnsmasq = DnsmasqClient.fromEnv(); - -// name -> address, primed silently so a restart does not re-announce every name it already answered. -const known = new Map(); -let primed = false; - -async function poll(): Promise { - const now = new Map((await dnsmasq.answeredNames()).map((a) => [a.name, a.address])); - if (primed) { - for (const [name, address] of now) { - if (!known.has(name)) await emit("name.added", { name, address }); - } - for (const [name] of known) { - if (!now.has(name)) await emit("name.removed", { name }); - } - } - known.clear(); - for (const [name, address] of now) known.set(name, address); - primed = true; -} - -const run = (): void => void poll().catch((err) => console.error(`[dnsmasq] ${err}`)); -setInterval(run, 30_000); -run(); - -console.log("[dnsmasq] watching the resolver's answered names"); diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 8fc98255..d868d771 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -17,7 +17,9 @@ ], "emits": [ "name.added", - "name.removed" + "name.removed", + "restarted", + "restart-judged" ], "own-secrets": { "broker": "${dir:mesh-state}/broker" @@ -28,6 +30,11 @@ "scope": "mesh" } ], + "tools": [ + "dnsmasq_health", + "dnsmasq_names", + "dnsmasq_resolve" + ], "listens": [ { "name": "dns-udp", @@ -63,7 +70,20 @@ "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n#\n# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the\n# `mesh-dns-resolver` seat, which more than one machine may hold, each answering\n# the same names from the same roster. Every node and every container lists every\n# holder and no public resolver, so whichever answers first gives the one answer.\n# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the\n# mesh's, rendered into each holder; no other copy lives on any machine.\n\n# What the mesh computed: one wildcard per machine — its name and everything\n# under it — and the mesh's own suffix as a local domain, so a name under it is\n# answered here or not at all and is never asked upstream. Rewritten whenever a\n# machine joins or leaves, which is why the service below restarts on it: a\n# reload makes dnsmasq re-read hosts files, not its configuration, and a\n# wildcard is configuration.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to\n# the private address of the machine its module runs on and the port its\n# answerer is published on. Nothing in a zone is answered here; what names a zone\n# holds is the module's. Rewritten whenever a zone is declared or moves, and the\n# service restarts on it for the same reason as above.\nconf-file=/etc/mesh-resolver/zones.conf\n\n# Where it answers: this machine's private address, so every node — and every\n# container on every node, which copies its machine's resolvers — can ask; and\n# loopback, for this machine's own use. Never a LAN address: a device that is\n# not a member cannot reach what the mesh's names point at, and a LAN's resolver\n# is its router's (ADR 0194).\n#\n# Named as an ADDRESS and not as the interface that carries it, on purpose. A\n# container's query is addressed to this address but arrives on the runtime's\n# bridge, and dnsmasq checks every query against what it was told to answer on:\n# an `interface=` line admits queries by the interface they arrive on, a\n# `listen-address=` line by the address they are sent to. With `interface=mesh0`\n# alone, a query from a container was received and dropped without a word\n# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on.\n#\n# bind-dynamic rather than bind-interfaces: the private address does not exist\n# until the machine is on the private network, and binding an address that is\n# not there yet fails to start rather than waiting for it.\nbind-dynamic\nlisten-address=${machine:address}\nlisten-address=127.0.0.1\n\n# **It must never read resolv.conf to find out where to forward.** This\n# machine's resolv.conf names this resolver — so a resolver that read it for\n# upstreams would find itself, and every query it could not answer locally would\n# loop until its receive queue filled. That is not theoretical: it filled with\n# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop\n# impossible: the upstreams are the two lines below, and nothing on the machine\n# can redirect them. The mesh's own names never reach them: the local= line in\n# the file above stops them here, answered or refused.\nno-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n\n# Both upstreams validate DNSSEC and say so with the Authenticated Data bit,\n# and this passes that bit down rather than dropping it, which dnsmasq does\n# unless told. It does not validate itself: an answer's trust is the upstream's\n# and the path to it, which is what a forwarding resolver's trust always was.\n# Without it a program that refuses to run behind a resolver that does not\n# validate — a mail system's admin does exactly that check at start — cannot\n# use this resolver (novox/hq 04-ISSUES/171).\nproxy-dnssec\n\n# A name without a dot is never forwarded, and reverse lookups of private ranges\n# are answered here rather than asking the world who 10.x is.\ndomain-needed\nbogus-priv\n\n# **No hosts file, and no operator's files.** This machine's /etc/hosts is its\n# own — the hosts module's block and the operator's lines (ADR 0199) — and the\n# mesh's resolver answers every node, so a line written for one machine's own\n# programs must not become an answer for all of them. Every per-node resolver\n# went wrong exactly here: a hosts file read once at start, an operator's old\n# line beside the mesh's, a drop-in read from a directory nobody owned.\nno-hosts\n" + "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n#\n# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the\n# `mesh-dns-resolver` seat, which more than one machine may hold, each answering\n# the same names from the same roster. Every node and every container lists every\n# holder and no public resolver, so whichever answers first gives the one answer.\n# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the\n# mesh's, rendered into each holder; no other copy lives on any machine.\n\n# What the mesh computed: one wildcard per machine — its name and everything\n# under it — and the mesh's own suffix as a local domain, so a name under it is\n# answered here or not at all and is never asked upstream. Rewritten whenever a\n# machine joins or leaves, which is why the service below restarts on it: a\n# reload makes dnsmasq re-read hosts files, not its configuration, and a\n# wildcard is configuration.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to\n# the private address of the machine its module runs on and the port its\n# answerer is published on. Nothing in a zone is answered here; what names a zone\n# holds is the module's. Rewritten whenever a zone is declared or moves, and the\n# service restarts on it for the same reason as above.\nconf-file=/etc/mesh-resolver/zones.conf\n\n# Where it answers: this machine's private address, so every node — and every\n# container on every node, which copies its machine's resolvers — can ask; and\n# loopback, for this machine's own use. Never a LAN address: a device that is\n# not a member cannot reach what the mesh's names point at, and a LAN's resolver\n# is its router's (ADR 0194).\n#\n# Named as an ADDRESS and not as the interface that carries it, on purpose. A\n# container's query is addressed to this address but arrives on the runtime's\n# bridge, and dnsmasq checks every query against what it was told to answer on:\n# an `interface=` line admits queries by the interface they arrive on, a\n# `listen-address=` line by the address they are sent to. With `interface=mesh0`\n# alone, a query from a container was received and dropped without a word\n# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on.\n#\n# bind-interfaces, and never bind-dynamic (novox/hq issue 348). bind-dynamic\n# re-reads the machine's addresses on every change of any link and closes each\n# listener whose interface that reading did not find; a reading made while the\n# container runtime deletes a bridge's virtual links fails (\"index gone\"), and\n# then every listener is closed until a later change reads cleanly. On\n# 2026-10-09 the private address was left unbound for twenty minutes, the daemon\n# running and silent. bind-interfaces binds each address once, at start, and\n# never closes it. It cannot bind an address that is not there yet, so the unit\n# waits for it (the drop-in below); a new private address restarts the unit, as\n# this file changing always did.\nbind-interfaces\nlisten-address=${machine:address}\nlisten-address=127.0.0.1\n\n# **It must never read resolv.conf to find out where to forward.** This\n# machine's resolv.conf names this resolver — so a resolver that read it for\n# upstreams would find itself, and every query it could not answer locally would\n# loop until its receive queue filled. That is not theoretical: it filled with\n# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop\n# impossible: the upstreams are the two lines below, and nothing on the machine\n# can redirect them. The mesh's own names never reach them: the local= line in\n# the file above stops them here, answered or refused.\nno-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n\n# Both upstreams validate DNSSEC and say so with the Authenticated Data bit,\n# and this passes that bit down rather than dropping it, which dnsmasq does\n# unless told. It does not validate itself: an answer's trust is the upstream's\n# and the path to it, which is what a forwarding resolver's trust always was.\n# Without it a program that refuses to run behind a resolver that does not\n# validate — a mail system's admin does exactly that check at start — cannot\n# use this resolver (novox/hq 04-ISSUES/171).\nproxy-dnssec\n\n# A name without a dot is never forwarded, and reverse lookups of private ranges\n# are answered here rather than asking the world who 10.x is.\ndomain-needed\nbogus-priv\n\n# **No hosts file, and no operator's files.** This machine's /etc/hosts is its\n# own — the hosts module's block and the operator's lines (ADR 0199) — and the\n# mesh's resolver answers every node, so a line written for one machine's own\n# programs must not become an answer for all of them. Every per-node resolver\n# went wrong exactly here: a hosts file read once at start, an operator's old\n# line beside the mesh's, a drop-in read from a directory nobody owned.\nno-hosts\n" + }, + { + "id": "drop-ins", + "type": "directory", + "path": "/etc/systemd/system/dnsmasq.service.d", + "mode": "0755" + }, + { + "id": "wait-for-address", + "type": "file", + "path": "/etc/systemd/system/dnsmasq.service.d/mesh.conf", + "mode": "0644", + "content": "# Written by the mesh (module dnsmasq, novox/hq issue 348). Replaced on every push.\n#\n# The resolver binds its addresses once, at start (bind-interfaces), so it starts only once this\n# machine's private address is there: it waits up to 80 seconds for it, and a start that found it\n# missing is tried again every ten seconds rather than left failed.\n[Unit]\nAfter=network-online.target\nWants=network-online.target\nStartLimitIntervalSec=0\n\n[Service]\nExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 80); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo \"${machine:address} is not on this machine\" >&2; exit 1'\nRestart=on-failure\nRestartSec=10s\n" }, { "id": "service", @@ -73,9 +93,37 @@ "boot": "enabled", "restart-on": [ "config", + "wait-for-address", "dnsmasq.fact-node-zones", "dnsmasq.fact-zones" - ] + ], + "health": { + "kind": "unit" + } + }, + { + "id": "watch-dir", + "type": "directory", + "path": "/var/lib/dnsmasq-watch", + "mode": "0755" + }, + { + "id": "watch", + "type": "process", + "name": "dnsmasq-watch", + "artifact": "tools", + "run": [ + "./dnsmasq-tools", + "watch" + ], + "health": { + "kind": "tool", + "tool": "dnsmasq_health", + "interval": "20s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + } } ], "facts": { @@ -87,5 +135,20 @@ "path": "/etc/mesh-resolver/zones.conf", "template": "# Generated by the mesh. Do not edit — this file is replaced whenever a module declares a zone or\n# its machine moves (novox/hq ADR 0199). One forwarding line per zone, to the module answering it.\n\n{{range .Zones}}server=/{{.Zone}}/{{.Address}}#{{.Port}}\n{{end}}" } + }, + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/dnsmasq-tools", + "binary": "dnsmasq-tools", + "loads": [ + "dnsmasq-tools" + ] + } + ] } } diff --git a/modules/dnsmasq/package.json b/modules/dnsmasq/package.json deleted file mode 100644 index dbf68350..00000000 --- a/modules/dnsmasq/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@novox/module-dnsmasq", - "version": "0.1.0", - "description": "dnsmasq — the mesh's resolver: answers wildcard node names and forwards the rest upstream. Its tools and events live here.", - "type": "module", - "private": true, - "dependencies": { "@novox/mesh-sdk": "^0.1.0" }, - "devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" } -} diff --git a/modules/dnsmasq/tools/index.ts b/modules/dnsmasq/tools/index.ts deleted file mode 100644 index bfeb7d8b..00000000 --- a/modules/dnsmasq/tools/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -// dnsmasq's tools — a resolver's two useful questions: what does it answer, and does it answer a -// given name. Moved into the module (novox/hq ADR 0039); the mesh serves them through the sdk. - -import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { DnsmasqClient } from "../client.js"; - -export function getDnsmasqTools(dnsmasq: DnsmasqClient): ToolDefinition[] { - return [ - { - name: "dnsmasq_names", - description: "The mesh names this node's resolver answers — each a machine and everything under it.", - input: {}, - run: async () => ({ names: await dnsmasq.answeredNames() }), - }, - { - name: "dnsmasq_resolve", - description: "Resolve a mesh name through this node's own resolver — the check that wildcard-resolution answers.", - input: { name: { type: "string", description: "a name to resolve, e.g. plex.anchor.internal" } }, - run: async (args) => { - const name = String(args.name); - try { - return { name, addresses: await dnsmasq.resolve(name) }; - } catch (err) { - return { name, addresses: [], error: err instanceof Error ? err.message : String(err) }; - } - }, - }, - ]; -} - -registerModuleTools("dnsmasq", (env) => getDnsmasqTools(DnsmasqClient.fromEnv(env))); diff --git a/modules/dnsmasq/tsconfig.json b/modules/dnsmasq/tsconfig.json deleted file mode 100644 index 36778592..00000000 --- a/modules/dnsmasq/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "noEmit": true - }, - "include": ["client.ts", "index.ts", "tools/index.ts"] -} diff --git a/modules/docker/cmd/docker-tools/docker.go b/modules/docker/cmd/docker-tools/docker.go index 872b8f3d..88669917 100644 --- a/modules/docker/cmd/docker-tools/docker.go +++ b/modules/docker/cmd/docker-tools/docker.go @@ -991,30 +991,53 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st } // Events is what the runtime did in a window ending now, the latest last. +// +// **The runtime keeps only its last events in memory** (about 256), and the images' own health checks +// make three exec events each per look: on a machine with a few checked containers that history is ten +// minutes long or less, all of it exec events (hq research 032). A window asked for further back than +// the history reaches is answered with what is still held AND says where the history begins, so an +// empty list never reads as "nothing happened" (hq issue 348: three hours asked across an outage, +// answered zero). The type filter is applied here rather than by the runtime, for the same reason: the +// history's start is only known from all of it. func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int, execs bool) (map[string]any, error) { - args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"} + return c.eventsAt(ctx, time.Now(), minutes, kind, limit, execs) +} + +func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind string, limit int, execs bool) (map[string]any, error) { if kind != "" { switch kind { case "container", "image", "network", "volume", "daemon", "plugin", "builder": default: return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind) } - args = append(args, "--filter", "type="+kind) } + // The whole of what the runtime holds, from the epoch: its buffer, which says where its history really + // begins and whether it is full; the window is cut from it here. + args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"} out, err := c.docker(ctx, args...) if err != nil { return nil, err } - raw, err := jsonLines[runtimeEvent](out) + all, err := jsonLines[runtimeEvent](out) if err != nil { return nil, err } + start := now.Add(-time.Duration(minutes) * time.Minute) + raw := make([]runtimeEvent, 0, len(all)) + for _, e := range all { + if !time.Unix(0, e.TimeNano).Before(start) { + raw = append(raw, e) + } + } // An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer // is read by agents and kept in their transcripts, which would make it a second copy of the leak. envs := &envCache{c: c, ctx: ctx} redacted := map[string]bool{} events := []map[string]any{} for _, e := range raw { + if kind != "" && e.Type != kind { + continue + } if !execs && strings.HasPrefix(e.Action, "exec_") { continue } @@ -1042,14 +1065,50 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int if len(events) > limit { events = events[len(events)-limit:] } - answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events} + answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events, + "held": len(all)} if len(redacted) > 0 { answer["redacted"] = sortedSet(redacted) answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)" } + for k, v := range historyReach(all, start) { + answer[k] = v + } return answer, nil } +// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer +// holding fewer has every event since the runtime started; a full one has lost what came before its oldest. +const RuntimeEventsKept = 256 + +// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window +// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not +// full — then it holds everything since the runtime started. Otherwise where the history begins, and that +// nothing before it can be answered from here. +func historyReach(all []runtimeEvent, start time.Time) map[string]any { + if len(all) == 0 { + return map[string]any{"complete": true, "oldest_held": nil, + "history": "the runtime holds no events at all: none happened since it started"} + } + oldest := time.Unix(0, all[0].TimeNano) + for _, e := range all { + if t := time.Unix(0, e.TimeNano); t.Before(oldest) { + oldest = t + } + } + at := oldest.UTC().Format(time.RFC3339) + if len(all) < RuntimeEventsKept { + return map[string]any{"complete": true, "oldest_held": at, + "history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)} + } + if !oldest.After(start) { + return map[string]any{"complete": true, "oldest_held": at} + } + return map[string]any{"complete": false, "oldest_held": at, + "history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)", + len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} +} + func sortedSet(set map[string]bool) []string { out := make([]string, 0, len(set)) for k := range set { diff --git a/modules/docker/cmd/docker-tools/docker_test.go b/modules/docker/cmd/docker-tools/docker_test.go index 22774e83..7bf8042f 100644 --- a/modules/docker/cmd/docker-tools/docker_test.go +++ b/modules/docker/cmd/docker-tools/docker_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io/fs" "os" "reflect" @@ -299,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { {"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2} ` f := (&fake{}).on("docker events", Ran{Stdout: out}) - got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false) if err != nil { t.Fatal(err) } @@ -307,7 +308,8 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" { t.Fatalf("%v", evs) } - if !f.ran("docker events --since 30m --until 0s --format {{json .}} --filter type=container") { + // The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348). + if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { t.Fatalf("%+v", f.calls) } if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil { @@ -362,3 +364,70 @@ func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) { t.Fatalf("%v %v", u, err) } } + +// eventLine is one line of `docker events --format '{{json .}}'`. +func eventLine(at time.Time, kind, action, name string) string { + return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n", + kind, action, name, at.UnixNano()) +} + +// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the +// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened. +func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) { + now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) + var out strings.Builder + for i := 0; i < RuntimeEventsKept; i++ { + out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail")) + } + got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false) + if err != nil { + t.Fatal(err) + } + if got["count"] != 0 || got["held"] != RuntimeEventsKept { + t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept) + } + if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") || + got["oldest_held"] != "2026-10-09T11:14:00Z" { + t.Fatalf("an empty answer from a short history reads as quiet: %v", got) + } +} + +func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) { + now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) + // A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer + // is not full, so it holds every event since the runtime started. + out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") + + eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + + eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") + + eventLine(now.Add(-1*time.Minute), "network", "create", "build-net") + f := (&fake{}).on("docker events", Ran{Stdout: out}) + got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false) + if err != nil { + t.Fatal(err) + } + if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") { + t.Fatalf("a quiet machine's window: %v", got) + } + if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { + t.Fatal("the type was filtered by the runtime, so the history's start is unknown") + } + // A quiet machine asked for ten minutes, its newest event an hour old: complete, and none. + got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("a quiet machine read as one whose history was lost: %v", got) + } + // No events at all: none since the runtime started. + got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("an empty buffer: %v", got) + } + // A full buffer whose oldest is before the window: complete. + var full strings.Builder + for i := 0; i < RuntimeEventsKept; i++ { + full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x")) + } + got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) + if got["complete"] != true || got["history"] != nil { + t.Fatalf("a full buffer reaching past the window: %v", got) + } +} diff --git a/modules/docker/cmd/docker-tools/main.go b/modules/docker/cmd/docker-tools/main.go index f5c812ea..66abc0bf 100644 --- a/modules/docker/cmd/docker-tools/main.go +++ b/modules/docker/cmd/docker-tools/main.go @@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool { return c.Inspect(ctx, ref) }, }, + { + Name: "docker_resolv_conf", + Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " + + "its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.", + Input: map[string]any{"container": containerArg}, + Run: func(args map[string]any) (any, error) { + ref, err := text(args, "container") + if err != nil { + return nil, err + } + return c.ResolvConfOf(ctx, ref) + }, + }, { Name: "docker_logs", Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " + @@ -262,7 +275,8 @@ func tools(c *Client) []stdio.Tool { { Name: "docker_events", Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " + - "an exec's command line is shown with any secret it carried as [redacted: ] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag.", + "an exec's command line is shown with any secret it carried as [redacted: ] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag. " + + "The runtime keeps only its last ~256 events in memory, which health-check execs fill in minutes: complete=false and history say where what it holds begins, and an empty list then means gone, not quiet.", Input: map[string]any{ "minutes": map[string]any{"type": "integer", "description": "how far back (default 60, at most 1440)"}, "type": map[string]any{"type": "string", "description": "only one kind: container, image, network, volume, daemon, plugin or builder"}, diff --git a/modules/docker/cmd/docker-tools/resolv.go b/modules/docker/cmd/docker-tools/resolv.go new file mode 100644 index 00000000..e04750fa --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv.go @@ -0,0 +1,111 @@ +package main + +// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's +// resolver file, made by the runtime when it is created and kept under the runtime's root-only +// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the +// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the +// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the +// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream): +// no exec into the container, nothing written, and it works where the program has no shell. + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "strings" +) + +// ContainerResolvConf is the path read inside a container. +const ContainerResolvConf = "/etc/resolv.conf" + +// HostResolvConf is the machine's own resolver file, read for comparison. +const HostResolvConf = "/etc/resolv.conf" + +// ResolvConf is a resolver file, parsed. +type ResolvConf struct { + Nameservers []string `json:"nameservers"` + Search []string `json:"search,omitempty"` + Options []string `json:"options,omitempty"` + Raw string `json:"raw"` +} + +// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only. +func parseResolvConf(text string) ResolvConf { + out := ResolvConf{Nameservers: []string{}, Raw: text} + for _, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") { + continue + } + switch f[0] { + case "nameserver": + if len(f) > 1 { + out.Nameservers = append(out.Nameservers, f[1]) + } + case "search", "domain": + out.Search = append(out.Search, f[1:]...) + case "options": + out.Options = append(out.Options, f[1:]...) + } + } + return out +} + +// fileFromArchive is the one regular file a `docker cp : -` stream carries. +func fileFromArchive(stream string) (string, error) { + tr := tar.NewReader(strings.NewReader(stream)) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + return "", errors.New("the runtime's archive held no file") + } + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + switch h.Typeflag { + case tar.TypeReg: + body, err := io.ReadAll(io.LimitReader(tr, 64<<10)) + if err != nil { + return "", fmt.Errorf("the runtime's archive could not be read: %w", err) + } + return string(body), nil + case tar.TypeSymlink: + return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname) + } + } +} + +// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list +// the same resolvers in the same order. +func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) { + ref, err := Ref(ref) + if err != nil { + return nil, err + } + answer := map[string]any{"container": ref, "path": ContainerResolvConf} + stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-") + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + text, err := fileFromArchive(stream) + if err != nil { + return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err) + } + inside := parseResolvConf(text) + answer["resolv_conf"] = inside + if raw, err := c.ReadFile(HostResolvConf); err != nil { + answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error() + } else { + host := parseResolvConf(string(raw)) + answer["machine_nameservers"] = host.Nameservers + same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ") + answer["same_as_machine"] = same + if !same { + answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " + + "or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again" + } + } + return answer, nil +} diff --git a/modules/docker/cmd/docker-tools/resolv_test.go b/modules/docker/cmd/docker-tools/resolv_test.go new file mode 100644 index 00000000..6c1285e8 --- /dev/null +++ b/modules/docker/cmd/docker-tools/resolv_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func archiveOf(t *testing.T, h tar.Header, body string) string { + t.Helper() + var buf bytes.Buffer + w := tar.NewWriter(&buf) + h.Size = int64(len(body)) + if h.Typeflag == tar.TypeSymlink { + h.Size = 0 + } + if err := w.WriteHeader(&h); err != nil { + t.Fatal(err) + } + if h.Typeflag != tar.TypeSymlink { + if _, err := w.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + w.Close() + return buf.String() +} + +// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy +// of the file could not be read by anything on the mesh. +func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) { + inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n" + f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -", + Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)}) + c := client(f, 1000) + c.ReadFile = func(p string) ([]byte, error) { + if p != HostResolvConf { + t.Fatalf("read %s", p) + } + return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil + } + got, err := c.ResolvConfOf(context.Background(), "route-proxy") + if err != nil { + t.Fatal(err) + } + rc := got["resolv_conf"].(ResolvConf) + if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" || + strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside { + t.Fatalf("parsed: %+v", rc) + } + if got["same_as_machine"] != false || got["note"] == nil { + t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got) + } + if f.ran("docker exec") { + t.Fatal("read by exec, not through the archive") + } +} + +func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) { + c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") { + t.Fatalf("a missing container: %v", err) + } + link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "") + c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") { + t.Fatalf("a link inside: %v", err) + } + c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000) + if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil { + t.Fatal("garbage was read as a file") + } + ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n") + c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000) + c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") } + got, err := c.ResolvConfOf(context.Background(), "x") + if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil { + t.Fatalf("an unreadable machine file: %v, %v", got, err) + } +} + +// The container is named as docker knows it, never as something docker would read as an option. +func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) { + f := &fake{} + for _, ref := range []string{"--privileged", "a b", "", "x;rm"} { + if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil { + t.Errorf("%q was asked of the runtime", ref) + } + } + if len(f.calls) != 0 { + t.Fatalf("the runtime was asked: %+v", f.calls) + } +} diff --git a/modules/docker/cmd/docker-tools/secrets_test.go b/modules/docker/cmd/docker-tools/secrets_test.go index ef2fa257..bd8baaf8 100644 --- a/modules/docker/cmd/docker-tools/secrets_test.go +++ b/modules/docker/cmd/docker-tools/secrets_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "strings" "testing" + "time" ) // The shape of the leak in hq issue 268: a server password announced at start and a database URI @@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) { f := (&fake{}). on("docker events", Ran{Stdout: execEvents}). on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"}) - got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true) if err != nil { t.Fatal(err) } diff --git a/modules/docker/module.json b/modules/docker/module.json index b7fc0726..73c8f74c 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -18,6 +18,7 @@ "tools": [ "docker_list", "docker_inspect", + "docker_resolv_conf", "docker_logs", "docker_secrets_in_logs", "docker_secrets_in_events", @@ -38,6 +39,11 @@ "docker_problems", "docker_ports" ], + "replaces": { + "docker_resolv_conf": [ + "docker exec cat /etc/resolv.conf" + ] + }, "resources": [ { "id": "package", diff --git a/modules/netcheck/cmd/netcheck/ask.go b/modules/netcheck/cmd/netcheck/ask.go new file mode 100644 index 00000000..ade2f085 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/ask.go @@ -0,0 +1,252 @@ +package main + +// Asking one named resolver directly (hq issue 348). netcheck_dns without a server asks the machine's +// resolver the way its programs do — resolv.conf, hosts, search domains, several servers in turn — which +// answers "can this machine resolve?" and hides which resolver answered. During the 2026-10-09 outage the +// question was "does the resolver at 10.10.0.1 answer?", and nothing on the mesh could ask it. Asked with a +// server, the question goes to that address and port alone, over UDP or TCP, and the answer says its +// response code: a refusal, no such name, and no answer at all are three different findings. +// +// The wire format is written here rather than taken from a library: a question and the records of five +// types are all it needs, and the module stays without a dependency. + +import ( + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" +) + +var dnsTypeCodes = map[string]uint16{"A": 1, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28} + +var rcodeWords = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"} + +// serverAddress is a server as asked — an address, with a port or without (53) — as host:port. +func serverAddress(server string) (string, error) { + server = strings.TrimSpace(server) + if host, port, err := net.SplitHostPort(server); err == nil { + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return "", fmt.Errorf("server %q: the port is not one (1-65535)", server) + } + if net.ParseIP(host) == nil { + return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server) + } + return net.JoinHostPort(host, port), nil + } + if net.ParseIP(server) == nil { + return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server) + } + return net.JoinHostPort(server, "53"), nil +} + +// askServer asks one resolver one question and reads its answer. +func askServer(server, name, kind string, overTCP bool, timeout time.Duration) (rcode int, answers []string, err error) { + addr, err := serverAddress(server) + if err != nil { + return 0, nil, err + } + query, id, err := buildQuery(name, dnsTypeCodes[kind]) + if err != nil { + return 0, nil, err + } + network := "udp" + if overTCP { + network = "tcp" + } + conn, err := net.DialTimeout(network, addr, timeout) + if err != nil { + return 0, nil, err + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(timeout)) + var reply []byte + if overTCP { + framed := binary.BigEndian.AppendUint16(nil, uint16(len(query))) + if _, err := conn.Write(append(framed, query...)); err != nil { + return 0, nil, err + } + var size [2]byte + if _, err := readFull(conn, size[:]); err != nil { + return 0, nil, err + } + reply = make([]byte, binary.BigEndian.Uint16(size[:])) + if _, err := readFull(conn, reply); err != nil { + return 0, nil, err + } + } else { + if _, err := conn.Write(query); err != nil { + return 0, nil, err + } + buf := make([]byte, 4096) + n, err := conn.Read(buf) + if err != nil { + return 0, nil, err + } + reply = buf[:n] + } + return parseReply(reply, id, dnsTypeCodes[kind]) +} + +func readFull(conn net.Conn, b []byte) (int, error) { + got := 0 + for got < len(b) { + n, err := conn.Read(b[got:]) + got += n + if err != nil { + return got, err + } + } + return got, nil +} + +// buildQuery is one question, recursion desired, under a random id. +func buildQuery(name string, qtype uint16) ([]byte, uint16, error) { + var idb [2]byte + _, _ = rand.Read(idb[:]) + id := binary.BigEndian.Uint16(idb[:]) + msg := binary.BigEndian.AppendUint16(nil, id) + msg = append(msg, 0x01, 0x00) // recursion desired + msg = append(msg, 0, 1, 0, 0, 0, 0, 0, 0) // one question + wire, err := encodeName(name) + if err != nil { + return nil, 0, err + } + msg = append(msg, wire...) + msg = binary.BigEndian.AppendUint16(msg, qtype) + msg = binary.BigEndian.AppendUint16(msg, 1) // IN + return msg, id, nil +} + +func encodeName(name string) ([]byte, error) { + name = strings.TrimSuffix(strings.TrimSpace(name), ".") + var out []byte + if name != "" { + for _, label := range strings.Split(name, ".") { + if label == "" || len(label) > 63 { + return nil, fmt.Errorf("name %q has an empty or too long label", name) + } + out = append(out, byte(len(label))) + out = append(out, label...) + } + } + return append(out, 0), nil +} + +var errShort = errors.New("the answer is cut short") + +// readName reads a possibly compressed name at off, and the offset after it. +func readName(msg []byte, off int) (string, int, error) { + var labels []string + next, jumped := -1, 0 + for { + if off >= len(msg) { + return "", 0, errShort + } + l := int(msg[off]) + switch { + case l == 0: + if next < 0 { + next = off + 1 + } + return strings.Join(labels, "."), next, nil + case l&0xC0 == 0xC0: + if off+1 >= len(msg) { + return "", 0, errShort + } + if next < 0 { + next = off + 2 + } + if jumped++; jumped > 32 { + return "", 0, errors.New("the answer's names point in a loop") + } + off = int(binary.BigEndian.Uint16(msg[off:]) & 0x3FFF) + default: + if off+1+l > len(msg) { + return "", 0, errShort + } + labels = append(labels, string(msg[off+1:off+1+l])) + off += 1 + l + } + } +} + +// parseReply reads the response code and the answers of the asked type. +func parseReply(msg []byte, id, qtype uint16) (int, []string, error) { + if len(msg) < 12 { + return 0, nil, errShort + } + if binary.BigEndian.Uint16(msg) != id { + return 0, nil, errors.New("the answer is to another question (its id differs)") + } + if msg[2]&0x80 == 0 { + return 0, nil, errors.New("the answer is not marked as one") + } + rcode := int(msg[3] & 0x0F) + qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:])) + off := 12 + for i := 0; i < qd; i++ { + _, n, err := readName(msg, off) + if err != nil { + return rcode, nil, err + } + off = n + 4 + } + answers := []string{} + for i := 0; i < an; i++ { + _, n, err := readName(msg, off) + if err != nil { + return rcode, answers, err + } + off = n + if off+10 > len(msg) { + return rcode, answers, errShort + } + rtype := binary.BigEndian.Uint16(msg[off:]) + size := int(binary.BigEndian.Uint16(msg[off+8:])) + data := off + 10 + if data+size > len(msg) { + return rcode, answers, errShort + } + rdata := msg[data : data+size] + off = data + size + if rtype != qtype { + continue // a CNAME on the way to an address, say + } + switch rtype { + case 1, 28: + answers = append(answers, net.IP(rdata).String()) + case 5: + s, _, err := readName(msg, data) + if err != nil { + return rcode, answers, err + } + answers = append(answers, s) + case 15: + if size < 3 { + return rcode, answers, errShort + } + s, _, err := readName(msg, data+2) + if err != nil { + return rcode, answers, err + } + answers = append(answers, fmt.Sprintf("%d %s", binary.BigEndian.Uint16(rdata), s)) + case 16: + var parts []string + for p := 0; p < len(rdata); { + l := int(rdata[p]) + if p+1+l > len(rdata) { + return rcode, answers, errShort + } + parts = append(parts, string(rdata[p+1:p+1+l])) + p += 1 + l + } + answers = append(answers, strings.Join(parts, "")) + } + } + return rcode, answers, nil +} diff --git a/modules/netcheck/cmd/netcheck/ask_test.go b/modules/netcheck/cmd/netcheck/ask_test.go new file mode 100644 index 00000000..1ad2c293 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/ask_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "encoding/binary" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode. +func respond(q []byte, rcode byte) []byte { + end := 12 + for q[end] != 0 { + end += int(q[end]) + 1 + } + question := q[12 : end+5] + name := strings.ToLower(string(q[13:end])) + r := append([]byte{}, q[:2]...) + r = append(r, 0x81, 0x80|rcode, 0, 1) + answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1 + if answer { + r = append(r, 0, 1, 0, 0, 0, 0) + } else { + r = append(r, 0, 0, 0, 0, 0, 0) + } + r = append(r, question...) + if answer { + r = append(r, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1) + } + return r +} + +func udpResolver(t *testing.T, rcode byte) string { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { pc.Close() }) + go func() { + buf := make([]byte, 512) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + _, _ = pc.WriteTo(respond(buf[:n], rcode), from) + } + }() + return pc.LocalAddr().String() +} + +func tcpResolver(t *testing.T) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + var size [2]byte + if _, err := readFull(c, size[:]); err == nil { + q := make([]byte, binary.BigEndian.Uint16(size[:])) + if _, err := readFull(c, q); err == nil { + r := respond(q, 0) + _, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...)) + } + } + c.Close() + } + }() + return l.Addr().String() +} + +// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer? +func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) { + got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second) + if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" { + t.Fatalf("udp: %+v, %v", got, err) + } + got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second) + if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" { + t.Fatalf("tcp: %+v, %v", got, err) + } + got, _ = AskDNS("nope.internal", "A", udpResolver(t, 3), false, time.Second) + if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 { + t.Fatalf("no such name is a result with its code: %+v", got) + } + got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second) + if got.Rcode != "REFUSED" || got.Error == "" { + t.Fatalf("a refusal: %+v", got) + } +} + +func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) { + l, _ := net.Listen("tcp", "127.0.0.1:0") + addr := l.Addr().String() + l.Close() + got, err := AskDNS("novox.internal", "A", addr, true, time.Second) + if err != nil || !strings.Contains(got.Error, "connection refused") { + t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err) + } + if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil { + t.Fatal("a server given by name was accepted") + } + if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil { + t.Fatal("tcp without a server was accepted") + } + if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" { + t.Fatalf("default port: %s", a) + } + for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53", + "[::1]:53": "[::1]:53", "::1": "[::1]:53"} { + if a, err := serverAddress(server); err != nil || a != want { + t.Errorf("%q: %q, %v; want %q", server, a, err, want) + } + } + for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536", + "10.10.0.1:dns", ""} { + if a, err := serverAddress(server); err == nil { + t.Errorf("%q was taken as %q", server, a) + } + } +} + +func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) { + root := t.TempDir() + must := func(err error) { + if err != nil { + t.Fatal(err) + } + } + must(os.MkdirAll(filepath.Join(root, "net"), 0o755)) + head := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + // 127.0.0.1:53 and 10.10.0.1:53 listening, a connection on 22 (not listening), and an udp 53 bound on loopback only. + must(os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(head+ + " 0: 0100007F:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 111 1 0 100 0 0 10 0\n"+ + " 1: 01000A0A:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 112 1 0 100 0 0 10 0\n"+ + " 2: 0100007F:0016 0200007F:9C40 01 00000000:00000000 00:00000000 00000000 0 0 113 1 0 100 0 0 10 0\n"), 0o644)) + must(os.WriteFile(filepath.Join(root, "net", "udp"), []byte(head+ + " 0: 0100007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000 0 0 121 2 0 0\n"), 0o644)) + must(os.MkdirAll(filepath.Join(root, "42", "fd"), 0o755)) + must(os.WriteFile(filepath.Join(root, "42", "comm"), []byte("dnsmasq\n"), 0o644)) + must(os.Symlink("socket:[111]", filepath.Join(root, "42", "fd", "5"))) + got := Listening(root, 53) + if got.Count != 3 || len(got.Unread) != 0 { + t.Fatalf("%+v", got) + } + var where []string + for _, s := range got.Sockets { + where = append(where, s.Protocol+" "+s.Address) + } + if strings.Join(where, ",") != "tcp 10.10.0.1,tcp 127.0.0.1,udp 127.0.0.1" { + t.Fatalf("sockets: %v", where) + } + if got.Sockets[1].Process != "42/dnsmasq" || got.Sockets[0].Owner == "" || !strings.Contains(got.Owners, "2 socket(s)") { + t.Fatalf("owners: %+v", got) + } + if missing := Listening(t.TempDir(), 0); len(missing.Unread) != 2 { + t.Fatalf("an unreadable table is said, not left out: %+v", missing) + } +} + +func TestListeningOnThisMachineFindsASocketItOpened(t *testing.T) { + if _, err := os.Stat("/proc/net/tcp"); err != nil { + t.Skip("no /proc here") + } + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + port := l.Addr().(*net.TCPAddr).Port + got := Listening("/proc", port) + if got.Count != 1 || got.Sockets[0].Address != "127.0.0.1" { + t.Fatalf("%+v", got) + } + if got.Sockets[0].Process == "" { + t.Fatalf("its own socket's program is not named: %+v", got) + } +} diff --git a/modules/netcheck/cmd/netcheck/checks.go b/modules/netcheck/cmd/netcheck/checks.go index 152b6e43..db7ca08c 100644 --- a/modules/netcheck/cmd/netcheck/checks.go +++ b/modules/netcheck/cmd/netcheck/checks.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "syscall" "time" ) @@ -50,8 +51,13 @@ func CheckTCP(host string, port int, timeout time.Duration) (TCPResult, error) { // DNSResult is what netcheck_dns answers. type DNSResult struct { - Name string `json:"name"` - Type string `json:"type"` + Name string `json:"name"` + Type string `json:"type"` + // Server and Protocol say which resolver was asked when one was named; empty, the machine's. + Server string `json:"server,omitempty"` + Protocol string `json:"protocol,omitempty"` + // Rcode is the named resolver's response code: NOERROR, NXDOMAIN, REFUSED, SERVFAIL… + Rcode string `json:"rcode,omitempty"` Answers []string `json:"answers"` ElapsedMS int64 `json:"elapsed_ms"` Error string `json:"error,omitempty"` @@ -64,6 +70,13 @@ var DNSTypes = []string{"A", "AAAA", "CNAME", "TXT", "MX"} // the machine's /etc/resolv.conf and /etc/hosts, which is the resolver this machine's programs use. // A name that does not resolve is a result with its error; an unknown type is refused. func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) { + return AskDNS(name, kind, "", false, timeout) +} + +// AskDNS is CheckDNS, or — with a server — the same question put to that resolver alone, over UDP or +// TCP, its hosts file and search domains left out (hq issue 348). No answer, a refusal and a response +// code other than NOERROR are results with their error; a malformed question is refused. +func AskDNS(name, kind, server string, overTCP bool, timeout time.Duration) (DNSResult, error) { kind = strings.ToUpper(strings.TrimSpace(kind)) if kind == "" { kind = "A" @@ -76,6 +89,37 @@ func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) { return DNSResult{}, fmt.Errorf("type %q is not one netcheck_dns looks up (%s)", kind, strings.Join(DNSTypes, ", ")) } out := DNSResult{Name: name, Type: kind, Answers: []string{}} + if server != "" { + if _, err := serverAddress(server); err != nil { + return DNSResult{}, err + } + out.Server, out.Protocol = server, "udp" + if overTCP { + out.Protocol = "tcp" + } + start := time.Now() + rcode, answers, err := askServer(server, name, kind, overTCP, timeout) + out.ElapsedMS = time.Since(start).Milliseconds() + if err == nil { + out.Rcode = rcodeWords[rcode] + if out.Rcode == "" { + out.Rcode = "RCODE" + strconv.Itoa(rcode) + } + if rcode != 0 { + out.Error = "the resolver answered " + out.Rcode + } + } + out.Answers = append(out.Answers, answers...) + if err != nil { + out.Error = dnsWords(err, timeout) + } + if kind != "MX" { + sort.Strings(out.Answers) + } + return out, nil + } else if overTCP { + return DNSResult{}, errors.New("tcp asks a named server; give server too") + } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() r := net.DefaultResolver @@ -171,3 +215,16 @@ func timeoutOf(args map[string]any) (time.Duration, error) { } return d, nil } + +// dnsWords is a failed question in words: refused (nothing listens there), no answer in time, or what the +// system said. +func dnsWords(err error, timeout time.Duration) string { + var ne net.Error + switch { + case errors.Is(err, syscall.ECONNREFUSED): + return "connection refused: nothing listens on that address and port" + case errors.As(err, &ne) && ne.Timeout(): + return fmt.Sprintf("no answer within %s", timeout) + } + return err.Error() +} diff --git a/modules/netcheck/cmd/netcheck/checks_test.go b/modules/netcheck/cmd/netcheck/checks_test.go index 8e2af867..fb770c77 100644 --- a/modules/netcheck/cmd/netcheck/checks_test.go +++ b/modules/netcheck/cmd/netcheck/checks_test.go @@ -57,12 +57,12 @@ func TestTimeoutIsDefaultedAndBounded(t *testing.T) { } } -func TestBothToolsAreListedUnprefixed(t *testing.T) { +func TestTheToolsAreListedUnprefixed(t *testing.T) { names := map[string]bool{} for _, tool := range tools() { names[tool.Name] = true } - if !names["netcheck_tcp"] || !names["netcheck_dns"] || len(names) != 2 { + if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 { t.Fatalf("tools: %v", names) } } diff --git a/modules/netcheck/cmd/netcheck/listening.go b/modules/netcheck/cmd/netcheck/listening.go new file mode 100644 index 00000000..cc5a8ac2 --- /dev/null +++ b/modules/netcheck/cmd/netcheck/listening.go @@ -0,0 +1,179 @@ +package main + +// What listens on this machine (hq issue 348): the kernel's own socket tables, read where `ss -lunt` reads +// them. During the 2026-10-09 outage the resolver's unit stayed active while it no longer listened on its +// private address, and nothing on the mesh could show that: whether a socket is bound is the kernel's to +// say, not the unit's. + +import ( + "bufio" + "encoding/hex" + "fmt" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Socket is one listening TCP socket or one bound UDP socket. +type Socket struct { + Protocol string `json:"protocol"` + Address string `json:"address"` + Port int `json:"port"` + // Process is the program holding it, "pid/name"; empty when Owner says why it is not known. + Process string `json:"process,omitempty"` + Owner string `json:"owner,omitempty"` + inode string +} + +// ListeningResult is what netcheck_listening answers. +type ListeningResult struct { + Sockets []Socket `json:"sockets"` + Count int `json:"count"` + // Unread names a table that could not be read: a socket in it is missing from the list, not absent. + Unread []string `json:"unread,omitempty"` + // Owners says how many sockets' programs are unknown and why, when any is. + Owners string `json:"owners,omitempty"` +} + +// Listening reads the socket tables under root (the machine's /proc), keeps TCP sockets in the listen +// state and every bound UDP socket, on port when port is not 0. +func Listening(root string, port int) ListeningResult { + out := ListeningResult{Sockets: []Socket{}} + for _, t := range []struct{ file, protocol string }{ + {"tcp", "tcp"}, {"tcp6", "tcp6"}, {"udp", "udp"}, {"udp6", "udp6"}, + } { + f, err := os.Open(filepath.Join(root, "net", t.file)) + if err != nil { + if !os.IsNotExist(err) || !strings.HasSuffix(t.file, "6") { + out.Unread = append(out.Unread, t.file+": "+err.Error()) + } + continue + } + sockets, err := parseSocketTable(f, t.protocol, port) + f.Close() + if err != nil { + out.Unread = append(out.Unread, t.file+": "+err.Error()) + } + out.Sockets = append(out.Sockets, sockets...) + } + unknown := owners(root, out.Sockets) + if unknown > 0 { + out.Owners = fmt.Sprintf("%d socket(s) without their program: held by a process of another account, whose "+ + "open files this tool may not read; the address and port are the kernel's and complete", unknown) + } + sort.Slice(out.Sockets, func(i, j int) bool { + a, b := out.Sockets[i], out.Sockets[j] + if a.Port != b.Port { + return a.Port < b.Port + } + if a.Protocol != b.Protocol { + return a.Protocol < b.Protocol + } + return a.Address < b.Address + }) + out.Count = len(out.Sockets) + return out +} + +// parseSocketTable reads one /proc/net table: "sl local_address rem_address st … inode". +func parseSocketTable(f *os.File, protocol string, port int) ([]Socket, error) { + var out []Socket + sc := bufio.NewScanner(f) + first := true + for sc.Scan() { + if first { + first = false + continue + } + fields := strings.Fields(sc.Text()) + if len(fields) < 10 { + continue + } + state := fields[3] + tcp := strings.HasPrefix(protocol, "tcp") + // TCP 0A is LISTEN; UDP 07 is a bound socket (unconnected), 01 a connected one, which is not listening. + if (tcp && state != "0A") || (!tcp && state != "07") { + continue + } + addr, p, err := hexAddress(fields[1]) + if err != nil { + return out, err + } + if port != 0 && p != port { + continue + } + out = append(out, Socket{Protocol: protocol, Address: addr, Port: p, inode: fields[9]}) + } + return out, sc.Err() +} + +// hexAddress reads the kernel's "0100007F:0035": the address in host order per 32-bit word, the port big-endian. +func hexAddress(s string) (string, int, error) { + a, p, ok := strings.Cut(s, ":") + if !ok { + return "", 0, fmt.Errorf("address %q", s) + } + port, err := strconv.ParseUint(p, 16, 16) + if err != nil { + return "", 0, fmt.Errorf("port %q", p) + } + raw, err := hex.DecodeString(a) + if err != nil || (len(raw) != 4 && len(raw) != 16) { + return "", 0, fmt.Errorf("address %q", a) + } + ip := make(net.IP, len(raw)) + for w := 0; w < len(raw); w += 4 { + ip[w], ip[w+1], ip[w+2], ip[w+3] = raw[w+3], raw[w+2], raw[w+1], raw[w] + } + return ip.String(), int(port), nil +} + +// owners names each socket's program from the processes' open files, as far as they can be read, and +// counts those it could not name. +func owners(root string, sockets []Socket) int { + want := map[string][]int{} + for i, s := range sockets { + want["socket:["+s.inode+"]"] = append(want["socket:["+s.inode+"]"], i) + } + if len(want) == 0 { + return 0 + } + procs, _ := os.ReadDir(root) + for _, p := range procs { + pid, err := strconv.Atoi(p.Name()) + if err != nil { + continue + } + fds, err := os.ReadDir(filepath.Join(root, p.Name(), "fd")) + if err != nil { + continue + } + for _, fd := range fds { + link, err := os.Readlink(filepath.Join(root, p.Name(), "fd", fd.Name())) + if err != nil { + continue + } + idx, ok := want[link] + if !ok { + continue + } + name, _ := os.ReadFile(filepath.Join(root, p.Name(), "comm")) + for _, i := range idx { + if sockets[i].Process == "" { + sockets[i].Process = fmt.Sprintf("%d/%s", pid, strings.TrimSpace(string(name))) + } + } + } + } + unknown := 0 + for i := range sockets { + if sockets[i].Process == "" { + sockets[i].Owner = "not readable by this tool's account" + unknown++ + } + } + return unknown +} diff --git a/modules/netcheck/cmd/netcheck/main.go b/modules/netcheck/cmd/netcheck/main.go index 98cb5074..99f14075 100644 --- a/modules/netcheck/cmd/netcheck/main.go +++ b/modules/netcheck/cmd/netcheck/main.go @@ -1,12 +1,14 @@ // netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches // and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's -// own sockets and resolver — a TCP connect and a DNS lookup — and nothing that changes anything. +// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named), +// and what listens here — and nothing that changes anything. // The module's HTTP check is its TypeScript bundle; the runtime serves both under one module. package main import ( "fmt" "os" + "strings" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -21,6 +23,29 @@ func main() { func tools() []stdio.Tool { return []stdio.Tool{ + { + Name: "netcheck_listening", + Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " + + "from the kernel's own tables (what ss -lunt reads), with the program holding it where this tool may see it. " + + "port narrows it to one port, e.g. 53 to see on which addresses the resolver answers.", + Input: map[string]any{ + "port": map[string]any{"type": "integer", "description": "only this port (default every port)"}, + }, + Run: func(args map[string]any) (any, error) { + port := 0 + if _, given := args["port"]; given { + p, err := whole(args, "port", 0) + if err != nil { + return nil, err + } + if p < 1 || p > 65535 { + return nil, fmt.Errorf("port %d is not a port (1-65535)", p) + } + port = p + } + return Listening("/proc", port), nil + }, + }, { Name: "netcheck_tcp", Description: "Check whether a TCP port is reachable from this machine: opens one connection " + @@ -48,11 +73,14 @@ func tools() []stdio.Tool { }, { Name: "netcheck_dns", - Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts). " + + Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " + + "that one resolver alone over UDP (or TCP), answering its response code (NOERROR, NXDOMAIN, REFUSED…). " + "type is A, AAAA, CNAME, TXT or MX; answers the records found, or the error.", Input: map[string]any{ "name": map[string]any{"type": "string", "description": "the name to look up"}, "type": map[string]any{"type": "string", "enum": []string{"A", "AAAA", "CNAME", "TXT", "MX"}, "description": "record type (default A)"}, + "server": map[string]any{"type": "string", "description": "ask this resolver alone: an address, with :port or without (53), e.g. 10.10.0.1"}, + "tcp": map[string]any{"type": "boolean", "description": "ask the server over TCP rather than UDP (default false)"}, "timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 3000, at most 30000)"}, }, Run: func(args map[string]any) (any, error) { @@ -65,7 +93,9 @@ func tools() []stdio.Tool { if err != nil { return nil, err } - return CheckDNS(name, kind, timeout) + server, _ := args["server"].(string) + overTCP, _ := args["tcp"].(bool) + return AskDNS(name, kind, strings.TrimSpace(server), overTCP, timeout) }, }, } diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json index 9ee8e8f5..c8f87901 100644 --- a/modules/netcheck/module.json +++ b/modules/netcheck/module.json @@ -4,8 +4,20 @@ "tools": [ "netcheck_tcp", "netcheck_dns", - "netcheck_http" + "netcheck_http", + "netcheck_listening" ], + "replaces": { + "netcheck_listening": [ + "ss -lunt", + "ss -ltn", + "netstat -lnt" + ], + "netcheck_dns": [ + "dig", + "nslookup" + ] + }, "build": { "artifacts": [ {