dnsmasq: ask the resolver a real question, and restart it when it stops answering (hq issue 348)
On 2026-10-09 the control node's dnsmasq stayed active and running, its journal silent, and refused every question on its private address for twenty minutes; nothing in the module looked past the unit. The module is now Go (its TypeScript tools were never served): a root watch asks the machine's own name on every listen address over UDP and TCP every 15s, restarts the unit after two failing looks (2 min apart, 3 an hour), and says whether that helped. dnsmasq_health is the watch process's declared check, so a restart that did not help becomes the module's unhealthy condition; the service is judged by its unit.
This commit is contained in:
jochen
@@ -0,0 +1,207 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A look asks the resolver one real question on every address its configuration tells it to answer on,
|
||||
// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat
|
||||
// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its
|
||||
// journal silent, and refused every question on its private address for twenty minutes; its unit said
|
||||
// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers.
|
||||
|
||||
// The files a look reads, under the machine's root.
|
||||
const (
|
||||
ConfigFile = "/etc/dnsmasq.conf"
|
||||
NodesFile = "/etc/mesh-resolver/nodes.conf"
|
||||
)
|
||||
|
||||
// AskWithin bounds each question.
|
||||
const AskWithin = 2 * time.Second
|
||||
|
||||
// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock.
|
||||
type Machine struct {
|
||||
Root string
|
||||
Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer
|
||||
Now func() time.Time
|
||||
// Port is where the resolver answers; 53 but in a test.
|
||||
Port string
|
||||
}
|
||||
|
||||
// Here is this machine.
|
||||
func Here() *Machine {
|
||||
return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"}
|
||||
}
|
||||
|
||||
func (m *Machine) read(path string) (string, error) {
|
||||
b, err := os.ReadFile(filepath.Join(m.Root, path))
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order.
|
||||
func ListenAddresses(config string) []string {
|
||||
var out []string
|
||||
sc := bufio.NewScanner(strings.NewReader(config))
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
value, ok := strings.CutPrefix(line, "listen-address=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, a := range strings.Split(value, ",") {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Named is a machine's name and its private address, as the roster gave them to the resolver.
|
||||
type Named struct {
|
||||
Name string `json:"name"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address:
|
||||
// its `host-record=<name>,<address>` lines.
|
||||
func AnsweredNames(nodes string) []Named {
|
||||
var out []Named
|
||||
sc := bufio.NewScanner(strings.NewReader(nodes))
|
||||
for sc.Scan() {
|
||||
value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, address, ok := strings.Cut(value, ",")
|
||||
if ok && name != "" && address != "" {
|
||||
out = append(out, Named{Name: name, Address: strings.TrimSpace(address)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Question is the name a look asks, and the address a right answer carries: the machine's own when the
|
||||
// roster names an address the resolver listens on, else the roster's first.
|
||||
func Question(listen []string, names []Named) (Named, bool) {
|
||||
for _, n := range names {
|
||||
for _, a := range listen {
|
||||
if n.Address == a {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return names[0], true
|
||||
}
|
||||
return Named{}, false
|
||||
}
|
||||
|
||||
// Look is one look: every listen address asked over both transports.
|
||||
type Look struct {
|
||||
At time.Time `json:"at"`
|
||||
// Healthy says every question was answered rightly.
|
||||
Healthy bool `json:"healthy"`
|
||||
Asked Named `json:"asked"`
|
||||
Answers []Answer `json:"answers"`
|
||||
// Wrong is every question not answered rightly, in words; Problems what kept the look from asking.
|
||||
Wrong []string `json:"wrong,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// Why is what is wrong in one sentence: empty when healthy.
|
||||
func (l Look) Why() string {
|
||||
if l.Healthy {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ")
|
||||
}
|
||||
|
||||
// Look asks every listen address its question, over UDP and TCP, all at once.
|
||||
func (m *Machine) Look(ctx context.Context) Look {
|
||||
l := Look{At: m.Now().UTC()}
|
||||
config, err := m.read(ConfigFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err))
|
||||
return l
|
||||
}
|
||||
listen := ListenAddresses(config)
|
||||
if len(listen) == 0 {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile))
|
||||
return l
|
||||
}
|
||||
nodes, err := m.read(NodesFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err))
|
||||
return l
|
||||
}
|
||||
q, ok := Question(listen, AnsweredNames(nodes))
|
||||
if !ok {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile))
|
||||
return l
|
||||
}
|
||||
l.Asked = q
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, address := range listen {
|
||||
for _, over := range []string{"udp", "tcp"} {
|
||||
wg.Add(1)
|
||||
go func(address, over string) {
|
||||
defer wg.Done()
|
||||
a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin)
|
||||
mu.Lock()
|
||||
l.Answers = append(l.Answers, a)
|
||||
mu.Unlock()
|
||||
}(address, over)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Slice(l.Answers, func(i, j int) bool {
|
||||
if l.Answers[i].Server != l.Answers[j].Server {
|
||||
return l.Answers[i].Server < l.Answers[j].Server
|
||||
}
|
||||
return l.Answers[i].Over > l.Answers[j].Over
|
||||
})
|
||||
for _, a := range l.Answers {
|
||||
if w := wrongWords(a, q); w != "" {
|
||||
l.Wrong = append(l.Wrong, w)
|
||||
}
|
||||
}
|
||||
l.Healthy = len(l.Wrong) == 0
|
||||
return l
|
||||
}
|
||||
|
||||
// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the
|
||||
// machine's address.
|
||||
func wrongWords(a Answer, q Named) string {
|
||||
at := a.Server + " over " + a.Over
|
||||
switch {
|
||||
case !a.Answered:
|
||||
return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed)
|
||||
case a.Code != "NOERROR":
|
||||
return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code)
|
||||
case a.Failed != "":
|
||||
return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed)
|
||||
}
|
||||
for _, x := range a.Addresses {
|
||||
if x == q.Address {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address)
|
||||
}
|
||||
|
||||
func orNothing(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "no address"
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
Reference in New Issue
Block a user