mesh-catalog/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go

package main

import (
	"context"
	"encoding/binary"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
// stopped — as dnsmasq answers the machine's own name from the roster.
type standIn struct {
	udp net.PacketConn
	tcp net.Listener
	wg  sync.WaitGroup
}

func reply(q []byte, address string) []byte {
	// The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name.
	out := append([]byte{}, q...)
	binary.BigEndian.PutUint16(out[2:], 0x8180)
	binary.BigEndian.PutUint16(out[6:], 1)
	ip := net.ParseIP(address).To4()
	out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4)
	return append(out, ip...)
}

func startStandIn(t *testing.T, at, port, address string) *standIn {
	t.Helper()
	s := &standIn{}
	var err error
	if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil {
		t.Fatal(err)
	}
	if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil {
		t.Fatal(err)
	}
	s.wg.Add(2)
	go func() {
		defer s.wg.Done()
		buf := make([]byte, 1500)
		for {
			n, from, err := s.udp.ReadFrom(buf)
			if err != nil {
				return
			}
			_, _ = s.udp.WriteTo(reply(buf[:n], address), from)
		}
	}()
	go func() {
		defer s.wg.Done()
		for {
			c, err := s.tcp.Accept()
			if err != nil {
				return
			}
			head := make([]byte, 2)
			if _, err := readFull(c, head); err == nil {
				q := make([]byte, binary.BigEndian.Uint16(head))
				if _, err := readFull(c, q); err == nil {
					r := reply(q, address)
					_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
				}
			}
			_ = c.Close()
		}
	}()
	t.Cleanup(s.stop)
	return s
}

func (s *standIn) stop() {
	_ = s.udp.Close()
	_ = s.tcp.Close()
	s.wg.Wait()
}

// freePort is a port free on both loopback addresses the tests use.
func freePort(t *testing.T) string {
	t.Helper()
	for i := 0; i < 20; i++ {
		l, err := net.Listen("tcp", "127.0.0.1:0")
		if err != nil {
			t.Fatal(err)
		}
		port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
		_ = l.Close()
		ok := true
		for _, at := range []string{"127.0.0.1", "127.0.0.2"} {
			for _, over := range []string{"tcp", "udp"} {
				if over == "tcp" {
					if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil {
						_ = c.Close()
					} else {
						ok = false
					}
				} else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil {
					_ = c.Close()
				} else {
					ok = false
				}
			}
		}
		if ok {
			return port
		}
	}
	t.Fatal("no port free on both loopback addresses")
	return ""
}

// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address)
// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2.
func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine {
	t.Helper()
	root := t.TempDir()
	write := func(path, text string) {
		p := filepath.Join(root, path)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n")
	write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+
		"address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n")
	if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil {
		t.Fatal(err)
	}
	return &Machine{Root: root, Ask: Ask, Now: now, Port: port}
}

func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) {
	listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n")
	if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" {
		t.Fatalf("listen addresses %v", listen)
	}
	names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" +
		"host-record=novox.internal,10.77.0.1\n")
	q, ok := Question(listen, names)
	if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" {
		t.Fatalf("asks %+v", q)
	}
	if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" {
		t.Fatalf("with no own name in the roster, asks %+v", q)
	}
}

// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback,
// and refused every question on its private address. Its unit said healthy. The look does not.
func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) {
	port := freePort(t)
	m := aResolverMachine(t, port, time.Now)
	private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" {
		t.Fatalf("a resolver answering everywhere: %+v", l)
	}
	private.stop()
	l := m.Look(t.Context())
	if l.Healthy || len(l.Wrong) != 2 {
		t.Fatalf("a resolver that stopped answering on its private address: %+v", l)
	}
	if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") ||
		!strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") {
		t.Fatalf("said as %q", why)
	}
}

func TestAWrongAnswerIsNotAnAnswer(t *testing.T) {
	q := Named{Name: "novox.internal", Address: "10.77.0.1"}
	for _, c := range []struct {
		a    Answer
		want string
	}{
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""},
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"},
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"},
		{Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"},
	} {
		if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
			t.Errorf("%+v said %q, want %q", c.a, got, c.want)
		}
	}
}

func failing(at time.Time) Look {
	return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}},
		Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}}
}

func healthy(at time.Time) Look { return Look{At: at, Healthy: true} }

// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks
// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why.
func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
	t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	var s State
	var why string
	if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 {
		t.Fatalf("one failing look restarted: %q", why)
	}
	if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
		t.Fatalf("a second failing look did not restart: %+v %q", s, why)
	}
	s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
	// It helped.
	s, why = Step(s, healthy(t0.Add(2*LookEvery)))
	if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
		t.Fatalf("a healthy look after a restart: %+v %q", s, why)
	}
	// Failing again soon after: held for the gap, then restarted; and so on until the budget is spent.
	at := t0.Add(3 * LookEvery)
	restarts := 1
	for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
		at = at.Add(LookEvery)
		s, why = Step(s, failing(at))
		if why == "" {
			continue
		}
		last := s.Restarts[len(s.Restarts)-1].At
		if at.Sub(last) < RestartGap {
			t.Fatalf("restarted %s after the last", at.Sub(last))
		}
		s.Restarts = append(s.Restarts, Restart{At: at, Why: why})
		restarts++
	}
	if restarts != RestartsPerHour {
		t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour)
	}
	if !strings.Contains(s.Held, "the most the watch makes") {
		t.Fatalf("the spent budget is not said: %+v", s)
	}
	for _, r := range s.Restarts[1:] {
		if r.Helped == nil || *r.Helped {
			t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped)
		}
	}
	// A look that could not ask — no configuration — restarts nothing.
	var none State
	broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
	none, _ = Step(none, broken)
	if _, why := Step(none, broken); why != "" {
		t.Fatalf("a missing configuration restarted the resolver: %q", why)
	}
}

// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the
// watch notices on its second look, restarts the unit (which brings the listener back), says the restart
// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not
// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the
// operator.
func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) {
	port := freePort(t)
	clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	var mu sync.Mutex
	now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
	pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() }
	m := aResolverMachine(t, port, now)
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	private.stop()

	fixes := true
	var asked []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		asked = append(asked, name+" "+strings.Join(args, " "))
		if fixes {
			private = startStandIn(t, "127.0.0.2", port, "127.0.0.2")
		}
		return "", nil
	}
	var s State
	look := func() {
		t.Helper()
		next, why := Step(s, m.Look(t.Context()))
		if why != "" {
			r := Restart{At: now().UTC(), Why: why}
			if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
				r.Error = err.Error()
			}
			next.Restarts = append(next.Restarts, r)
		}
		s = next
		if err := m.WriteState(s); err != nil {
			t.Fatal(err)
		}
		pass(LookEvery)
	}
	look()
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") {
		t.Fatalf("after one failing look: %+v", h.Why)
	}
	look()
	if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" {
		t.Fatalf("restarted with %v", asked)
	}
	look()
	if r := s.Restarts[0]; r.Helped == nil || !*r.Helped {
		t.Fatalf("the restart that brought it back is said as %+v", r)
	}
	if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 {
		t.Fatalf("after the restart helped: %+v", h)
	}

	// It stops again, and this time a restart does not help.
	private.stop()
	fixes = false
	pass(RestartGap)
	for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ {
		look()
	}
	h := m.Health(t.Context())
	if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") ||
		!strings.Contains(h.Why, "which did not help") {
		t.Fatalf("a restart that did not help: %q", h.Why)
	}
	// And a watch that stopped looking is said, whatever the resolver does.
	pass(Stale + time.Minute)
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") {
		t.Fatalf("a stale watch: %q", h.Why)
	}
}

func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) {
	var a Announcer
	t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
	old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}}
	if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 {
		t.Fatalf("said at its start: %+v", got)
	}
	if got := a.Next(nil, false, old); len(got) != 0 {
		t.Fatalf("an unreadable roster said %+v", got)
	}
	now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})}
	got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now)
	types := map[string]int{}
	for _, e := range got {
		types[e.Type]++
	}
	if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 {
		t.Fatalf("said %+v", got)
	}
}