mesh-catalog/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go

package main

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
)

// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
// stopped — as dnsmasq answers the machine's own name from the roster.
type standIn struct {
	udp net.PacketConn
	tcp net.Listener
	wg  sync.WaitGroup
}

func reply(q []byte, address string) []byte {
	// The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name.
	out := append([]byte{}, q...)
	binary.BigEndian.PutUint16(out[2:], 0x8180)
	binary.BigEndian.PutUint16(out[6:], 1)
	ip := net.ParseIP(address).To4()
	out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4)
	return append(out, ip...)
}

func startStandIn(t *testing.T, at, port, address string) *standIn {
	t.Helper()
	s := &standIn{}
	var err error
	if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil {
		t.Fatal(err)
	}
	if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil {
		t.Fatal(err)
	}
	s.wg.Add(2)
	go func() {
		defer s.wg.Done()
		buf := make([]byte, 1500)
		for {
			n, from, err := s.udp.ReadFrom(buf)
			if err != nil {
				return
			}
			_, _ = s.udp.WriteTo(reply(buf[:n], address), from)
		}
	}()
	go func() {
		defer s.wg.Done()
		for {
			c, err := s.tcp.Accept()
			if err != nil {
				return
			}
			head := make([]byte, 2)
			if _, err := readFull(c, head); err == nil {
				q := make([]byte, binary.BigEndian.Uint16(head))
				if _, err := readFull(c, q); err == nil {
					r := reply(q, address)
					_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
				}
			}
			_ = c.Close()
		}
	}()
	t.Cleanup(s.stop)
	return s
}

func (s *standIn) stop() {
	_ = s.udp.Close()
	_ = s.tcp.Close()
	s.wg.Wait()
}

// freePort is a port free on both loopback addresses the tests use.
func freePort(t *testing.T) string {
	t.Helper()
	for i := 0; i < 20; i++ {
		l, err := net.Listen("tcp", "127.0.0.1:0")
		if err != nil {
			t.Fatal(err)
		}
		port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
		_ = l.Close()
		ok := true
		for _, at := range []string{"127.0.0.1", "127.0.0.2"} {
			for _, over := range []string{"tcp", "udp"} {
				if over == "tcp" {
					if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil {
						_ = c.Close()
					} else {
						ok = false
					}
				} else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil {
					_ = c.Close()
				} else {
					ok = false
				}
			}
		}
		if ok {
			return port
		}
	}
	t.Fatal("no port free on both loopback addresses")
	return ""
}

// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address)
// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2.
func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine {
	t.Helper()
	root := t.TempDir()
	write := func(path, text string) {
		p := filepath.Join(root, path)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
			t.Fatal(err)
		}
	}
	write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n")
	write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+
		"address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n")
	if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil {
		t.Fatal(err)
	}
	return &Machine{Root: root, Ask: Ask, Now: now, Port: port}
}

func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) {
	listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n")
	if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" {
		t.Fatalf("listen addresses %v", listen)
	}
	names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" +
		"host-record=novox.internal,10.77.0.1\n")
	q, ok := Question(listen, names)
	if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" {
		t.Fatalf("asks %+v", q)
	}
	if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" {
		t.Fatalf("with no own name in the roster, asks %+v", q)
	}
}

// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback,
// and refused every question on its private address. Its unit said healthy. The look does not.
func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) {
	port := freePort(t)
	m := aResolverMachine(t, port, time.Now)
	private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" {
		t.Fatalf("a resolver answering everywhere: %+v", l)
	}
	private.stop()
	l := m.Look(t.Context())
	if l.Healthy || len(l.Wrong) != 2 {
		t.Fatalf("a resolver that stopped answering on its private address: %+v", l)
	}
	if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") ||
		!strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") {
		t.Fatalf("said as %q", why)
	}
}

func TestAWrongAnswerIsNotAnAnswer(t *testing.T) {
	q := Named{Name: "novox.internal", Address: "10.77.0.1"}
	for _, c := range []struct {
		a    Answer
		want string
	}{
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""},
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"},
		{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"},
		{Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"},
	} {
		if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
			t.Errorf("%+v said %q, want %q", c.a, got, c.want)
		}
	}
}

func failing(at time.Time) Look {
	return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}},
		Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}}
}

func healthy(at time.Time) Look { return Look{At: at, Healthy: true} }

// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks
// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why.
func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
	t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	var s State
	var why string
	if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 {
		t.Fatalf("one failing look restarted: %q", why)
	}
	if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
		t.Fatalf("a second failing look did not restart: %+v %q", s, why)
	}
	s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
	// It helped.
	s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active")
	if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
		t.Fatalf("a healthy look after a restart: %+v %q", s, why)
	}
	// Failing again soon after: held for the gap, then restarted; and so on until the budget is spent.
	at := t0.Add(3 * LookEvery)
	restarts := 1
	for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
		at = at.Add(LookEvery)
		s, why = Step(s, failing(at), "active")
		if why == "" {
			continue
		}
		last := s.Restarts[len(s.Restarts)-1].At
		if at.Sub(last) < RestartGap {
			t.Fatalf("restarted %s after the last", at.Sub(last))
		}
		s.Restarts = append(s.Restarts, Restart{At: at, Why: why})
		restarts++
	}
	if restarts != RestartsPerHour {
		t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour)
	}
	if !strings.Contains(s.Held, "the most the watch makes") {
		t.Fatalf("the spent budget is not said: %+v", s)
	}
	for _, r := range s.Restarts[1:] {
		if r.Helped == nil || *r.Helped {
			t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped)
		}
	}
	// A look that could not ask — no configuration — restarts nothing.
	var none State
	broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
	none, _ = Step(none, broken, "active")
	if _, why := Step(none, broken, "active"); why != "" {
		t.Fatalf("a missing configuration restarted the resolver: %q", why)
	}
}

// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the
// watch notices on its second look, restarts the unit (which brings the listener back), says the restart
// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not
// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the
// operator.
func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) {
	port := freePort(t)
	clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	var mu sync.Mutex
	now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
	pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() }
	m := aResolverMachine(t, port, now)
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	private.stop()

	fixes := true
	var asked []string
	run := func(_ context.Context, name string, args ...string) (string, error) {
		asked = append(asked, name+" "+strings.Join(args, " "))
		if fixes {
			private = startStandIn(t, "127.0.0.2", port, "127.0.0.2")
		}
		return "", nil
	}
	var s State
	look := func() {
		t.Helper()
		next, why := Step(s, m.Look(t.Context()), "active")
		if why != "" {
			r := Restart{At: now().UTC(), Why: why}
			if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
				r.Error = err.Error()
			}
			next.Restarts = append(next.Restarts, r)
		}
		s = next
		if err := m.WriteState(s); err != nil {
			t.Fatal(err)
		}
		pass(LookEvery)
	}
	look()
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") {
		t.Fatalf("after one failing look: %+v", h.Why)
	}
	look()
	if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" {
		t.Fatalf("restarted with %v", asked)
	}
	look()
	if r := s.Restarts[0]; r.Helped == nil || !*r.Helped {
		t.Fatalf("the restart that brought it back is said as %+v", r)
	}
	if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 {
		t.Fatalf("after the restart helped: %+v", h)
	}

	// It stops again, and this time a restart does not help.
	private.stop()
	fixes = false
	pass(RestartGap)
	for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ {
		look()
	}
	h := m.Health(t.Context())
	if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") ||
		!strings.Contains(h.Why, "which did not help") {
		t.Fatalf("a restart that did not help: %q", h.Why)
	}
	// And a watch that stopped looking is said, whatever the resolver does.
	pass(Stale + time.Minute)
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") {
		t.Fatalf("a stale watch: %q", h.Why)
	}
}

// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came
// back on its own, or by something else.
func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) {
	t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	s := State{Restarts: []Restart{{At: t0}}}
	s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active")
	if h := s.Restarts[0].Helped; h == nil || *h {
		t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h)
	}
	s = State{Restarts: []Restart{{At: t0}}}
	s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active")
	if h := s.Restarts[0].Helped; h == nil || !*h {
		t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h)
	}
}

// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is
// held, said, and not counted against the budget (review of PR 161, C2).
func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) {
	t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	for _, unit := range []string{"inactive", "activating", "failed", ""} {
		var s State
		var why string
		for i := 0; i < 4; i++ {
			s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit)
			if why != "" {
				t.Fatalf("a unit %q was restarted", unit)
			}
		}
		if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 {
			t.Fatalf("a unit %q: %+v", unit, s)
		}
		// Running again and still not answering: restarted, the held looks counted as failing.
		if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" {
			t.Fatalf("a unit running again after %q, not answering, was not restarted", unit)
		}
	}
}

// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts
// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4).
func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) {
	port := freePort(t)
	clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
	m := aResolverMachine(t, port, func() time.Time { return clock })
	m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer {
		return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"}
	}
	before := LookEvery
	LookEvery = time.Millisecond
	t.Cleanup(func() { LookEvery = before })
	run := func(spent []Restart) ([]string, State) {
		t.Helper()
		if err := m.WriteState(State{Restarts: spent}); err != nil {
			t.Fatal(err)
		}
		var asked []string
		ctx, cancel := context.WithCancel(t.Context())
		defer cancel()
		looks := 0
		var last State
		err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) {
			asked = append(asked, name+" "+strings.Join(args, " "))
			if args[0] == "show" {
				return "active\n", nil
			}
			return "", nil
		}, func(s State) error {
			last = s
			if looks++; looks == 3 {
				cancel()
			}
			return m.WriteState(s)
		})
		if err != nil {
			t.Fatal(err)
		}
		return asked, last
	}
	// Three restarts within the hour before it was restarted: it restarts nothing more.
	spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}}
	asked, last := run(spent)
	for _, a := range asked {
		if strings.Contains(a, "restart") {
			t.Fatalf("restarted with its budget spent before it started: %v", asked)
		}
	}
	if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 {
		t.Fatalf("the budget it kept: %+v", last)
	}
	// None spent: restarted on its second failing look, not waiting for the start.
	asked, last = run(nil)
	restarts := 0
	for _, a := range asked {
		if strings.HasPrefix(a, "systemctl restart") {
			restarts++
			if a != "systemctl restart --no-block dnsmasq.service" {
				t.Fatalf("restarted as %q", a)
			}
		}
	}
	if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" {
		t.Fatalf("restarts %d: %v %+v", restarts, asked, last)
	}
}

// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running.
func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) {
	port := freePort(t)
	clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
	m := aResolverMachine(t, port, func() time.Time { return clock })
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") {
		t.Fatalf("no watch: %+v", h.Why)
	}
	if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil {
		t.Fatal(err)
	}
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") {
		t.Fatalf("a stale watch: %+v", h.Why)
	}
	if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil {
		t.Fatal(err)
	}
	if h := m.Health(t.Context()); !h.Healthy || h.Why != "" {
		t.Fatalf("a watch looking, a resolver answering: %+v", h)
	}
	if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil {
		t.Fatal(err)
	}
	if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") {
		t.Fatalf("a unit stopped: %+v", h.Why)
	}
	if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil {
		t.Fatal(err)
	}
	if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") {
		t.Fatalf("an unreadable state: %+v", h.Why)
	}
}

// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3).
func TestResolveAsksOnlyAListenAddress(t *testing.T) {
	port := freePort(t)
	m := aResolverMachine(t, port, time.Now)
	startStandIn(t, "127.0.0.2", port, "127.0.0.2")
	startStandIn(t, "127.0.0.1", port, "127.0.0.2")
	if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") {
		t.Fatalf("another address was asked: %v", err)
	}
	got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp")
	if err != nil {
		t.Fatal(err)
	}
	answers := got.(map[string]any)["answers"].([]Answer)
	if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" {
		t.Fatalf("%+v", answers)
	}
	if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 {
		t.Fatalf("every listen address over both: %+v", got)
	}
	if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil {
		t.Fatal("over quic was accepted")
	}
	if _, err := m.Resolve(t.Context(), "", "", ""); err == nil {
		t.Fatal("no name was accepted")
	}
}

// The announcer: names primed silently, restarts made before the bundle started said, an event remembered
// only once emitted, and whether a restart helped said (review of PR 161, C5).
func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) {
	port := freePort(t)
	t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
	m := aResolverMachine(t, port, func() time.Time { return t0 })
	yes := true
	if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil {
		t.Fatal(err)
	}
	var said []string
	failing := true
	emit := func(kind string, body any, _ ...stdio.EmitOptions) error {
		if failing && kind == "restarted" {
			return errors.New("no bus")
		}
		said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"]))
		return nil
	}
	var a Announcer
	Tell(m, &a, emit)
	if strings.Join(said, "|") != "restart-judged <nil> true" {
		t.Fatalf("at the start: %v", said)
	}
	failing, said = false, nil
	Tell(m, &a, emit)
	if strings.Join(said, "|") != "restarted <nil> <nil>" {
		t.Fatalf("a restart whose emit failed was not said again: %v", said)
	}
	said = nil
	Tell(m, &a, emit)
	if len(said) != 0 {
		t.Fatalf("said twice: %v", said)
	}
	// The roster changes: one machine added, one gone. An unreadable roster says nothing.
	nodes := filepath.Join(m.Root, NodesFile)
	if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	Tell(m, &a, emit)
	if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") ||
		!strings.Contains(got, "name.removed ace.internal") || len(said) != 2 {
		t.Fatalf("the roster's change: %v", said)
	}
	said = nil
	if err := os.Remove(nodes); err != nil {
		t.Fatal(err)
	}
	Tell(m, &a, emit)
	if len(said) != 0 {
		t.Fatalf("an unreadable roster said %v", said)
	}
}