package main import ( "context" "encoding/binary" "errors" "fmt" "net" "os" "path/filepath" "strconv" "strings" "sync" "testing" "time" stdio "git.novox.be/novox/mesh-sdk/go" ) // A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until // stopped — as dnsmasq answers the machine's own name from the roster. type standIn struct { udp net.PacketConn tcp net.Listener wg sync.WaitGroup } func reply(q []byte, address string) []byte { // The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name. out := append([]byte{}, q...) binary.BigEndian.PutUint16(out[2:], 0x8180) binary.BigEndian.PutUint16(out[6:], 1) ip := net.ParseIP(address).To4() out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4) return append(out, ip...) } func startStandIn(t *testing.T, at, port, address string) *standIn { t.Helper() s := &standIn{} var err error if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil { t.Fatal(err) } if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil { t.Fatal(err) } s.wg.Add(2) go func() { defer s.wg.Done() buf := make([]byte, 1500) for { n, from, err := s.udp.ReadFrom(buf) if err != nil { return } _, _ = s.udp.WriteTo(reply(buf[:n], address), from) } }() go func() { defer s.wg.Done() for { c, err := s.tcp.Accept() if err != nil { return } head := make([]byte, 2) if _, err := readFull(c, head); err == nil { q := make([]byte, binary.BigEndian.Uint16(head)) if _, err := readFull(c, q); err == nil { r := reply(q, address) _, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...)) } } _ = c.Close() } }() t.Cleanup(s.stop) return s } func (s *standIn) stop() { _ = s.udp.Close() _ = s.tcp.Close() s.wg.Wait() } // freePort is a port free on both loopback addresses the tests use. func freePort(t *testing.T) string { t.Helper() for i := 0; i < 20; i++ { l, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port) _ = l.Close() ok := true for _, at := range []string{"127.0.0.1", "127.0.0.2"} { for _, over := range []string{"tcp", "udp"} { if over == "tcp" { if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil { _ = c.Close() } else { ok = false } } else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil { _ = c.Close() } else { ok = false } } } if ok { return port } } t.Fatal("no port free on both loopback addresses") return "" } // aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address) // and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2. func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine { t.Helper() root := t.TempDir() write := func(path, text string) { p := filepath.Join(root, path) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(p, []byte(text), 0o644); err != nil { t.Fatal(err) } } write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n") write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+ "address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n") if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil { t.Fatal(err) } return &Machine{Root: root, Ask: Ask, Now: now, Port: port} } func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) { listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n") if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" { t.Fatalf("listen addresses %v", listen) } names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" + "host-record=novox.internal,10.77.0.1\n") q, ok := Question(listen, names) if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" { t.Fatalf("asks %+v", q) } if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" { t.Fatalf("with no own name in the roster, asks %+v", q) } } // **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback, // and refused every question on its private address. Its unit said healthy. The look does not. func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) { port := freePort(t) m := aResolverMachine(t, port, time.Now) private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") startStandIn(t, "127.0.0.1", port, "127.0.0.2") if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" { t.Fatalf("a resolver answering everywhere: %+v", l) } private.stop() l := m.Look(t.Context()) if l.Healthy || len(l.Wrong) != 2 { t.Fatalf("a resolver that stopped answering on its private address: %+v", l) } if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") || !strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") { t.Fatalf("said as %q", why) } } func TestAWrongAnswerIsNotAnAnswer(t *testing.T) { q := Named{Name: "novox.internal", Address: "10.77.0.1"} for _, c := range []struct { a Answer want string }{ {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""}, {Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"}, {Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"}, {Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"}, } { if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) { t.Errorf("%+v said %q, want %q", c.a, got, c.want) } } } func failing(at time.Time) Look { return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}}, Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}} } func healthy(at time.Time) Look { return Look{At: at, Healthy: true} } // The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks // after it; waits RestartGap before another; and stops at RestartsPerHour, saying why. func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) var s State var why string if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 { t.Fatalf("one failing look restarted: %q", why) } if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { t.Fatalf("a second failing look did not restart: %+v %q", s, why) } s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why}) // It helped. s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active") if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped { t.Fatalf("a healthy look after a restart: %+v %q", s, why) } // Failing again soon after: held for the gap, then restarted; and so on until the budget is spent. at := t0.Add(3 * LookEvery) restarts := 1 for at.Add(LookEvery).Sub(t0) < 50*time.Minute { at = at.Add(LookEvery) s, why = Step(s, failing(at), "active") if why == "" { continue } last := s.Restarts[len(s.Restarts)-1].At if at.Sub(last) < RestartGap { t.Fatalf("restarted %s after the last", at.Sub(last)) } s.Restarts = append(s.Restarts, Restart{At: at, Why: why}) restarts++ } if restarts != RestartsPerHour { t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour) } if !strings.Contains(s.Held, "the most the watch makes") { t.Fatalf("the spent budget is not said: %+v", s) } for _, r := range s.Restarts[1:] { if r.Helped == nil || *r.Helped { t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped) } } // A look that could not ask — no configuration — restarts nothing. var none State broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}} none, _ = Step(none, broken, "active") if _, why := Step(none, broken, "active"); why != "" { t.Fatalf("a missing configuration restarted the resolver: %q", why) } } // **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the // watch notices on its second look, restarts the unit (which brings the listener back), says the restart // helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not // help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the // operator. func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) { port := freePort(t) clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) var mu sync.Mutex now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock } pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() } m := aResolverMachine(t, port, now) startStandIn(t, "127.0.0.1", port, "127.0.0.2") private := startStandIn(t, "127.0.0.2", port, "127.0.0.2") private.stop() fixes := true var asked []string run := func(_ context.Context, name string, args ...string) (string, error) { asked = append(asked, name+" "+strings.Join(args, " ")) if fixes { private = startStandIn(t, "127.0.0.2", port, "127.0.0.2") } return "", nil } var s State look := func() { t.Helper() next, why := Step(s, m.Look(t.Context()), "active") if why != "" { r := Restart{At: now().UTC(), Why: why} if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil { r.Error = err.Error() } next.Restarts = append(next.Restarts, r) } s = next if err := m.WriteState(s); err != nil { t.Fatal(err) } pass(LookEvery) } look() if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") { t.Fatalf("after one failing look: %+v", h.Why) } look() if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" { t.Fatalf("restarted with %v", asked) } look() if r := s.Restarts[0]; r.Helped == nil || !*r.Helped { t.Fatalf("the restart that brought it back is said as %+v", r) } if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 { t.Fatalf("after the restart helped: %+v", h) } // It stops again, and this time a restart does not help. private.stop() fixes = false pass(RestartGap) for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ { look() } h := m.Health(t.Context()) if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") || !strings.Contains(h.Why, "which did not help") { t.Fatalf("a restart that did not help: %q", h.Why) } // And a watch that stopped looking is said, whatever the resolver does. pass(Stale + time.Minute) if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") { t.Fatalf("a stale watch: %q", h.Why) } } // A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came // back on its own, or by something else. func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) { t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) s := State{Restarts: []Restart{{At: t0}}} s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active") if h := s.Restarts[0].Helped; h == nil || *h { t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h) } s = State{Restarts: []Restart{{At: t0}}} s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active") if h := s.Restarts[0].Helped; h == nil || !*h { t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h) } } // A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is // held, said, and not counted against the budget (review of PR 161, C2). func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) { t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) for _, unit := range []string{"inactive", "activating", "failed", ""} { var s State var why string for i := 0; i < 4; i++ { s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit) if why != "" { t.Fatalf("a unit %q was restarted", unit) } } if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 { t.Fatalf("a unit %q: %+v", unit, s) } // Running again and still not answering: restarted, the held looks counted as failing. if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" { t.Fatalf("a unit running again after %q, not answering, was not restarted", unit) } } } // The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts // without waiting for the start, and writes what it saw (review of PR 161, C2 and C4). func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) { port := freePort(t) clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) m := aResolverMachine(t, port, func() time.Time { return clock }) m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer { return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"} } before := LookEvery LookEvery = time.Millisecond t.Cleanup(func() { LookEvery = before }) run := func(spent []Restart) ([]string, State) { t.Helper() if err := m.WriteState(State{Restarts: spent}); err != nil { t.Fatal(err) } var asked []string ctx, cancel := context.WithCancel(t.Context()) defer cancel() looks := 0 var last State err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) { asked = append(asked, name+" "+strings.Join(args, " ")) if args[0] == "show" { return "active\n", nil } return "", nil }, func(s State) error { last = s if looks++; looks == 3 { cancel() } return m.WriteState(s) }) if err != nil { t.Fatal(err) } return asked, last } // Three restarts within the hour before it was restarted: it restarts nothing more. spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}} asked, last := run(spent) for _, a := range asked { if strings.Contains(a, "restart") { t.Fatalf("restarted with its budget spent before it started: %v", asked) } } if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 { t.Fatalf("the budget it kept: %+v", last) } // None spent: restarted on its second failing look, not waiting for the start. asked, last = run(nil) restarts := 0 for _, a := range asked { if strings.HasPrefix(a, "systemctl restart") { restarts++ if a != "systemctl restart --no-block dnsmasq.service" { t.Fatalf("restarted as %q", a) } } } if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" { t.Fatalf("restarts %d: %v %+v", restarts, asked, last) } } // The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running. func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) { port := freePort(t) clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) m := aResolverMachine(t, port, func() time.Time { return clock }) startStandIn(t, "127.0.0.1", port, "127.0.0.2") startStandIn(t, "127.0.0.2", port, "127.0.0.2") if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") { t.Fatalf("no watch: %+v", h.Why) } if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil { t.Fatal(err) } if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") { t.Fatalf("a stale watch: %+v", h.Why) } if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil { t.Fatal(err) } if h := m.Health(t.Context()); !h.Healthy || h.Why != "" { t.Fatalf("a watch looking, a resolver answering: %+v", h) } if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil { t.Fatal(err) } if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") { t.Fatalf("a unit stopped: %+v", h.Why) } if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil { t.Fatal(err) } if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") { t.Fatalf("an unreadable state: %+v", h.Why) } } // dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3). func TestResolveAsksOnlyAListenAddress(t *testing.T) { port := freePort(t) m := aResolverMachine(t, port, time.Now) startStandIn(t, "127.0.0.2", port, "127.0.0.2") startStandIn(t, "127.0.0.1", port, "127.0.0.2") if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") { t.Fatalf("another address was asked: %v", err) } got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp") if err != nil { t.Fatal(err) } answers := got.(map[string]any)["answers"].([]Answer) if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" { t.Fatalf("%+v", answers) } if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 { t.Fatalf("every listen address over both: %+v", got) } if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil { t.Fatal("over quic was accepted") } if _, err := m.Resolve(t.Context(), "", "", ""); err == nil { t.Fatal("no name was accepted") } } // The announcer: names primed silently, restarts made before the bundle started said, an event remembered // only once emitted, and whether a restart helped said (review of PR 161, C5). func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) { port := freePort(t) t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) m := aResolverMachine(t, port, func() time.Time { return t0 }) yes := true if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil { t.Fatal(err) } var said []string failing := true emit := func(kind string, body any, _ ...stdio.EmitOptions) error { if failing && kind == "restarted" { return errors.New("no bus") } said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"])) return nil } var a Announcer Tell(m, &a, emit) if strings.Join(said, "|") != "restart-judged true" { t.Fatalf("at the start: %v", said) } failing, said = false, nil Tell(m, &a, emit) if strings.Join(said, "|") != "restarted " { t.Fatalf("a restart whose emit failed was not said again: %v", said) } said = nil Tell(m, &a, emit) if len(said) != 0 { t.Fatalf("said twice: %v", said) } // The roster changes: one machine added, one gone. An unreadable roster says nothing. nodes := filepath.Join(m.Root, NodesFile) if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil { t.Fatal(err) } Tell(m, &a, emit) if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") || !strings.Contains(got, "name.removed ace.internal") || len(said) != 2 { t.Fatalf("the roster's change: %v", said) } said = nil if err := os.Remove(nodes); err != nil { t.Fatal(err) } Tell(m, &a, emit) if len(said) != 0 { t.Fatalf("an unreadable roster said %v", said) } }