dnsmasq: ask the resolver a real question, and restart it when it stops answering (hq issue 348)
On 2026-10-09 the control node's dnsmasq stayed active and running, its journal silent, and refused every question on its private address for twenty minutes; nothing in the module looked past the unit. The module is now Go (its TypeScript tools were never served): a root watch asks the machine's own name on every listen address over UDP and TCP every 15s, restarts the unit after two failing looks (2 min apart, 3 an hour), and says whether that helped. dnsmasq_health is the watch process's declared check, so a restart that did not help becomes the module's unhealthy condition; the service is judged by its unit.
This commit is contained in:
jochen
+1
-1
@@ -1,4 +1,4 @@
|
||||
70
|
||||
69
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
// dnsmasq's own code, living in the module (novox/hq ADR 0039). dnsmasq here answers the mesh's
|
||||
// generated wildcard names (<service>.<node>.<suffix>) itself and forwards everything else to fixed
|
||||
// upstreams — the predecessor's arrangement, so the machine and its containers ask it for the world.
|
||||
// Its code reads what it was told to answer, and can resolve through itself to prove that it does.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Resolver } from "node:dns/promises";
|
||||
|
||||
export interface AnsweredName {
|
||||
/** A machine's internal name, e.g. "anchor.internal" — it and everything under it resolve here. */
|
||||
name: string;
|
||||
address: string;
|
||||
}
|
||||
|
||||
export class DnsmasqClient {
|
||||
constructor(
|
||||
private readonly resolverPath: string,
|
||||
private readonly address: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Build from the environment. Both values are node-local facts with mesh-chosen defaults — the
|
||||
* wildcard file the module is sent, and the loopback address its config listens on (127.0.0.1,
|
||||
* where the predecessor's resolver answered and every machine's resolv.conf already points) — so
|
||||
* there is nothing to be unconfigured about; it never throws.
|
||||
*/
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): DnsmasqClient {
|
||||
return new DnsmasqClient(
|
||||
env.MESH_DNSMASQ_RESOLVER_PATH ?? "/etc/mesh-resolver/nodes.conf",
|
||||
env.MESH_DNSMASQ_ADDRESS ?? "127.0.0.1",
|
||||
);
|
||||
}
|
||||
|
||||
/** The names this resolver answers, read from the mesh-generated wildcard file. */
|
||||
async answeredNames(): Promise<AnsweredName[]> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(this.resolverPath, "utf8");
|
||||
} catch {
|
||||
return []; // not yet on the network, or the file has not been written — no names, not an error
|
||||
}
|
||||
const names: AnsweredName[] = [];
|
||||
for (const line of text.split("\n")) {
|
||||
// dnsmasq wildcard syntax the mesh writes: address=/<node>.<suffix>/<address>. The file also
|
||||
// carries local=/<suffix>/, which names no machine and is not matched here.
|
||||
const match = line.match(/^address=\/([^/]+)\/(.+)$/);
|
||||
if (match) names.push({ name: match[1], address: match[2] });
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** Resolve a name through this node's own resolver — the check that wildcard-resolution answers. */
|
||||
async resolve(name: string): Promise<string[]> {
|
||||
const resolver = new Resolver();
|
||||
resolver.setServers([this.address]);
|
||||
try {
|
||||
return await resolver.resolve4(name);
|
||||
} catch {
|
||||
return await resolver.resolve6(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A real question to the resolver, as any program on the machine asks it: one name, one record type, over
|
||||
// UDP or TCP, to one address. Written out here rather than taken from a library: the module needs the
|
||||
// header's answer code and the A records, nothing else, and Go's own resolver would read /etc/hosts and
|
||||
// /etc/resolv.conf instead of asking the one address it is told (novox/hq issue 348).
|
||||
|
||||
// Answer is what one question got.
|
||||
type Answer struct {
|
||||
// Server is where it was asked, and Over the transport.
|
||||
Server string `json:"server"`
|
||||
Over string `json:"over"`
|
||||
Name string `json:"name"`
|
||||
// Answered says a reply came; Code is its answer code (NOERROR, NXDOMAIN, …) and Addresses its A
|
||||
// records.
|
||||
Answered bool `json:"answered"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
// Failed is why no reply came, in words: refused (nothing listens there), no answer within, or what
|
||||
// the system said.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
// Took is how long it took, in milliseconds.
|
||||
Took int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
// rcodes are the answer codes by number.
|
||||
var rcodes = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
|
||||
|
||||
// query is one question for name's A record, with id.
|
||||
func query(name string, id uint16) ([]byte, error) {
|
||||
name = strings.TrimSuffix(name, ".")
|
||||
if name == "" {
|
||||
return nil, errors.New("no name to ask")
|
||||
}
|
||||
msg := make([]byte, 12, 64)
|
||||
binary.BigEndian.PutUint16(msg[0:], id)
|
||||
binary.BigEndian.PutUint16(msg[2:], 0x0100) // a standard query, recursion desired
|
||||
binary.BigEndian.PutUint16(msg[4:], 1) // one question
|
||||
for _, label := range strings.Split(name, ".") {
|
||||
if label == "" || len(label) > 63 {
|
||||
return nil, fmt.Errorf("%q is not a name that can be asked", name)
|
||||
}
|
||||
msg = append(msg, byte(len(label)))
|
||||
msg = append(msg, label...)
|
||||
}
|
||||
msg = append(msg, 0, 0, 1, 0, 1) // root; type A; class IN
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// readAnswer reads a reply to the question with id: its code and A records.
|
||||
func readAnswer(msg []byte, id uint16) (string, []string, error) {
|
||||
if len(msg) < 12 {
|
||||
return "", nil, errors.New("a reply too short to be one")
|
||||
}
|
||||
if binary.BigEndian.Uint16(msg[0:]) != id {
|
||||
return "", nil, errors.New("a reply to another question")
|
||||
}
|
||||
flags := binary.BigEndian.Uint16(msg[2:])
|
||||
code, ok := rcodes[int(flags&0xF)]
|
||||
if !ok {
|
||||
code = fmt.Sprintf("RCODE%d", flags&0xF)
|
||||
}
|
||||
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
|
||||
at := 12
|
||||
for i := 0; i < qd; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return code, nil, err
|
||||
}
|
||||
at += 4
|
||||
}
|
||||
var addresses []string
|
||||
for i := 0; i < an; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return code, addresses, err
|
||||
}
|
||||
if at+10 > len(msg) {
|
||||
return code, addresses, errors.New("a reply cut short")
|
||||
}
|
||||
kind, length := binary.BigEndian.Uint16(msg[at:]), int(binary.BigEndian.Uint16(msg[at+8:]))
|
||||
at += 10
|
||||
if at+length > len(msg) {
|
||||
return code, addresses, errors.New("a reply cut short")
|
||||
}
|
||||
if kind == 1 && length == 4 {
|
||||
addresses = append(addresses, net.IP(msg[at:at+4]).String())
|
||||
}
|
||||
at += length
|
||||
}
|
||||
return code, addresses, nil
|
||||
}
|
||||
|
||||
// skipName steps over a name in a message, a pointer ending it.
|
||||
func skipName(msg []byte, at int) (int, error) {
|
||||
for at < len(msg) {
|
||||
n := int(msg[at])
|
||||
switch {
|
||||
case n == 0:
|
||||
return at + 1, nil
|
||||
case n&0xC0 == 0xC0:
|
||||
return at + 2, nil
|
||||
}
|
||||
at += n + 1
|
||||
}
|
||||
return at, errors.New("a name runs past the reply")
|
||||
}
|
||||
|
||||
// Ask asks server (an address, port 53 unless given) for name's A record over "udp" or "tcp", within
|
||||
// timeout.
|
||||
func Ask(ctx context.Context, server, over, name string, timeout time.Duration) Answer {
|
||||
if _, _, err := net.SplitHostPort(server); err != nil {
|
||||
server = net.JoinHostPort(server, "53")
|
||||
}
|
||||
a := Answer{Server: server, Over: over, Name: name}
|
||||
started := time.Now()
|
||||
defer func() { a.Took = time.Since(started).Milliseconds() }()
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
id := uint16(rand.UintN(1 << 16))
|
||||
q, err := query(name, id)
|
||||
if err != nil {
|
||||
a.Failed = err.Error()
|
||||
return a
|
||||
}
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, over, server)
|
||||
if err != nil {
|
||||
a.Failed = failedWords(err, timeout)
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
defer conn.Close()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(deadline)
|
||||
}
|
||||
var reply []byte
|
||||
if over == "tcp" {
|
||||
framed := binary.BigEndian.AppendUint16(nil, uint16(len(q)))
|
||||
if _, err = conn.Write(append(framed, q...)); err == nil {
|
||||
head := make([]byte, 2)
|
||||
if _, err = readFull(conn, head); err == nil {
|
||||
reply = make([]byte, binary.BigEndian.Uint16(head))
|
||||
_, err = readFull(conn, reply)
|
||||
}
|
||||
}
|
||||
} else if _, err = conn.Write(q); err == nil {
|
||||
buf := make([]byte, 1500)
|
||||
var n int
|
||||
n, err = conn.Read(buf)
|
||||
reply = buf[:n]
|
||||
}
|
||||
if err != nil {
|
||||
a.Failed = failedWords(err, timeout)
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
a.Answered = true
|
||||
a.Code, a.Addresses, err = readAnswer(reply, id)
|
||||
if err != nil {
|
||||
a.Failed = err.Error()
|
||||
}
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
|
||||
func readFull(conn net.Conn, buf []byte) (int, error) {
|
||||
read := 0
|
||||
for read < len(buf) {
|
||||
n, err := conn.Read(buf[read:])
|
||||
read += n
|
||||
if err != nil {
|
||||
return read, err
|
||||
}
|
||||
}
|
||||
return read, nil
|
||||
}
|
||||
|
||||
// failedWords is why no reply came: refused when nothing listens, no answer when nothing came in time.
|
||||
func failedWords(err error, timeout time.Duration) string {
|
||||
var timedOut interface{ Timeout() bool }
|
||||
switch {
|
||||
case errors.Is(err, syscall.ECONNREFUSED):
|
||||
return "refused: nothing listens there"
|
||||
case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timedOut) && timedOut.Timeout()):
|
||||
return fmt.Sprintf("no answer within %s", timeout)
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
|
||||
// stopped — as dnsmasq answers the machine's own name from the roster.
|
||||
type standIn struct {
|
||||
udp net.PacketConn
|
||||
tcp net.Listener
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func reply(q []byte, address string) []byte {
|
||||
// The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name.
|
||||
out := append([]byte{}, q...)
|
||||
binary.BigEndian.PutUint16(out[2:], 0x8180)
|
||||
binary.BigEndian.PutUint16(out[6:], 1)
|
||||
ip := net.ParseIP(address).To4()
|
||||
out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4)
|
||||
return append(out, ip...)
|
||||
}
|
||||
|
||||
func startStandIn(t *testing.T, at, port, address string) *standIn {
|
||||
t.Helper()
|
||||
s := &standIn{}
|
||||
var err error
|
||||
if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.wg.Add(2)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := s.udp.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = s.udp.WriteTo(reply(buf[:n], address), from)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
c, err := s.tcp.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
head := make([]byte, 2)
|
||||
if _, err := readFull(c, head); err == nil {
|
||||
q := make([]byte, binary.BigEndian.Uint16(head))
|
||||
if _, err := readFull(c, q); err == nil {
|
||||
r := reply(q, address)
|
||||
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
|
||||
}
|
||||
}
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(s.stop)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *standIn) stop() {
|
||||
_ = s.udp.Close()
|
||||
_ = s.tcp.Close()
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
// freePort is a port free on both loopback addresses the tests use.
|
||||
func freePort(t *testing.T) string {
|
||||
t.Helper()
|
||||
for i := 0; i < 20; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
|
||||
_ = l.Close()
|
||||
ok := true
|
||||
for _, at := range []string{"127.0.0.1", "127.0.0.2"} {
|
||||
for _, over := range []string{"tcp", "udp"} {
|
||||
if over == "tcp" {
|
||||
if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil {
|
||||
_ = c.Close()
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
} else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil {
|
||||
_ = c.Close()
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return port
|
||||
}
|
||||
}
|
||||
t.Fatal("no port free on both loopback addresses")
|
||||
return ""
|
||||
}
|
||||
|
||||
// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address)
|
||||
// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2.
|
||||
func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
write := func(path, text string) {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n")
|
||||
write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+
|
||||
"address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n")
|
||||
if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &Machine{Root: root, Ask: Ask, Now: now, Port: port}
|
||||
}
|
||||
|
||||
func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) {
|
||||
listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n")
|
||||
if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" {
|
||||
t.Fatalf("listen addresses %v", listen)
|
||||
}
|
||||
names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" +
|
||||
"host-record=novox.internal,10.77.0.1\n")
|
||||
q, ok := Question(listen, names)
|
||||
if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" {
|
||||
t.Fatalf("asks %+v", q)
|
||||
}
|
||||
if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" {
|
||||
t.Fatalf("with no own name in the roster, asks %+v", q)
|
||||
}
|
||||
}
|
||||
|
||||
// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback,
|
||||
// and refused every question on its private address. Its unit said healthy. The look does not.
|
||||
func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) {
|
||||
port := freePort(t)
|
||||
m := aResolverMachine(t, port, time.Now)
|
||||
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" {
|
||||
t.Fatalf("a resolver answering everywhere: %+v", l)
|
||||
}
|
||||
private.stop()
|
||||
l := m.Look(t.Context())
|
||||
if l.Healthy || len(l.Wrong) != 2 {
|
||||
t.Fatalf("a resolver that stopped answering on its private address: %+v", l)
|
||||
}
|
||||
if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") ||
|
||||
!strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") {
|
||||
t.Fatalf("said as %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWrongAnswerIsNotAnAnswer(t *testing.T) {
|
||||
q := Named{Name: "novox.internal", Address: "10.77.0.1"}
|
||||
for _, c := range []struct {
|
||||
a Answer
|
||||
want string
|
||||
}{
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""},
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"},
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"},
|
||||
{Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"},
|
||||
} {
|
||||
if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
|
||||
t.Errorf("%+v said %q, want %q", c.a, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func failing(at time.Time) Look {
|
||||
return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}},
|
||||
Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}}
|
||||
}
|
||||
|
||||
func healthy(at time.Time) Look { return Look{At: at, Healthy: true} }
|
||||
|
||||
// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks
|
||||
// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why.
|
||||
func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
var s State
|
||||
var why string
|
||||
if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 {
|
||||
t.Fatalf("one failing look restarted: %q", why)
|
||||
}
|
||||
if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
|
||||
t.Fatalf("a second failing look did not restart: %+v %q", s, why)
|
||||
}
|
||||
s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
|
||||
// It helped.
|
||||
s, why = Step(s, healthy(t0.Add(2*LookEvery)))
|
||||
if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
|
||||
t.Fatalf("a healthy look after a restart: %+v %q", s, why)
|
||||
}
|
||||
// Failing again soon after: held for the gap, then restarted; and so on until the budget is spent.
|
||||
at := t0.Add(3 * LookEvery)
|
||||
restarts := 1
|
||||
for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
|
||||
at = at.Add(LookEvery)
|
||||
s, why = Step(s, failing(at))
|
||||
if why == "" {
|
||||
continue
|
||||
}
|
||||
last := s.Restarts[len(s.Restarts)-1].At
|
||||
if at.Sub(last) < RestartGap {
|
||||
t.Fatalf("restarted %s after the last", at.Sub(last))
|
||||
}
|
||||
s.Restarts = append(s.Restarts, Restart{At: at, Why: why})
|
||||
restarts++
|
||||
}
|
||||
if restarts != RestartsPerHour {
|
||||
t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour)
|
||||
}
|
||||
if !strings.Contains(s.Held, "the most the watch makes") {
|
||||
t.Fatalf("the spent budget is not said: %+v", s)
|
||||
}
|
||||
for _, r := range s.Restarts[1:] {
|
||||
if r.Helped == nil || *r.Helped {
|
||||
t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped)
|
||||
}
|
||||
}
|
||||
// A look that could not ask — no configuration — restarts nothing.
|
||||
var none State
|
||||
broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
|
||||
none, _ = Step(none, broken)
|
||||
if _, why := Step(none, broken); why != "" {
|
||||
t.Fatalf("a missing configuration restarted the resolver: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the
|
||||
// watch notices on its second look, restarts the unit (which brings the listener back), says the restart
|
||||
// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not
|
||||
// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the
|
||||
// operator.
|
||||
func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) {
|
||||
port := freePort(t)
|
||||
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
var mu sync.Mutex
|
||||
now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
|
||||
pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() }
|
||||
m := aResolverMachine(t, port, now)
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
private.stop()
|
||||
|
||||
fixes := true
|
||||
var asked []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||
if fixes {
|
||||
private = startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
var s State
|
||||
look := func() {
|
||||
t.Helper()
|
||||
next, why := Step(s, m.Look(t.Context()))
|
||||
if why != "" {
|
||||
r := Restart{At: now().UTC(), Why: why}
|
||||
if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
|
||||
r.Error = err.Error()
|
||||
}
|
||||
next.Restarts = append(next.Restarts, r)
|
||||
}
|
||||
s = next
|
||||
if err := m.WriteState(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pass(LookEvery)
|
||||
}
|
||||
look()
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") {
|
||||
t.Fatalf("after one failing look: %+v", h.Why)
|
||||
}
|
||||
look()
|
||||
if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" {
|
||||
t.Fatalf("restarted with %v", asked)
|
||||
}
|
||||
look()
|
||||
if r := s.Restarts[0]; r.Helped == nil || !*r.Helped {
|
||||
t.Fatalf("the restart that brought it back is said as %+v", r)
|
||||
}
|
||||
if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 {
|
||||
t.Fatalf("after the restart helped: %+v", h)
|
||||
}
|
||||
|
||||
// It stops again, and this time a restart does not help.
|
||||
private.stop()
|
||||
fixes = false
|
||||
pass(RestartGap)
|
||||
for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ {
|
||||
look()
|
||||
}
|
||||
h := m.Health(t.Context())
|
||||
if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") ||
|
||||
!strings.Contains(h.Why, "which did not help") {
|
||||
t.Fatalf("a restart that did not help: %q", h.Why)
|
||||
}
|
||||
// And a watch that stopped looking is said, whatever the resolver does.
|
||||
pass(Stale + time.Minute)
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") {
|
||||
t.Fatalf("a stale watch: %q", h.Why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) {
|
||||
var a Announcer
|
||||
t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
|
||||
old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}}
|
||||
if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 {
|
||||
t.Fatalf("said at its start: %+v", got)
|
||||
}
|
||||
if got := a.Next(nil, false, old); len(got) != 0 {
|
||||
t.Fatalf("an unreadable roster said %+v", got)
|
||||
}
|
||||
now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})}
|
||||
got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now)
|
||||
types := map[string]int{}
|
||||
for _, e := range got {
|
||||
types[e.Type]++
|
||||
}
|
||||
if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 {
|
||||
t.Fatalf("said %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A look asks the resolver one real question on every address its configuration tells it to answer on,
|
||||
// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat
|
||||
// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its
|
||||
// journal silent, and refused every question on its private address for twenty minutes; its unit said
|
||||
// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers.
|
||||
|
||||
// The files a look reads, under the machine's root.
|
||||
const (
|
||||
ConfigFile = "/etc/dnsmasq.conf"
|
||||
NodesFile = "/etc/mesh-resolver/nodes.conf"
|
||||
)
|
||||
|
||||
// AskWithin bounds each question.
|
||||
const AskWithin = 2 * time.Second
|
||||
|
||||
// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock.
|
||||
type Machine struct {
|
||||
Root string
|
||||
Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer
|
||||
Now func() time.Time
|
||||
// Port is where the resolver answers; 53 but in a test.
|
||||
Port string
|
||||
}
|
||||
|
||||
// Here is this machine.
|
||||
func Here() *Machine {
|
||||
return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"}
|
||||
}
|
||||
|
||||
func (m *Machine) read(path string) (string, error) {
|
||||
b, err := os.ReadFile(filepath.Join(m.Root, path))
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order.
|
||||
func ListenAddresses(config string) []string {
|
||||
var out []string
|
||||
sc := bufio.NewScanner(strings.NewReader(config))
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
value, ok := strings.CutPrefix(line, "listen-address=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, a := range strings.Split(value, ",") {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Named is a machine's name and its private address, as the roster gave them to the resolver.
|
||||
type Named struct {
|
||||
Name string `json:"name"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address:
|
||||
// its `host-record=<name>,<address>` lines.
|
||||
func AnsweredNames(nodes string) []Named {
|
||||
var out []Named
|
||||
sc := bufio.NewScanner(strings.NewReader(nodes))
|
||||
for sc.Scan() {
|
||||
value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, address, ok := strings.Cut(value, ",")
|
||||
if ok && name != "" && address != "" {
|
||||
out = append(out, Named{Name: name, Address: strings.TrimSpace(address)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Question is the name a look asks, and the address a right answer carries: the machine's own when the
|
||||
// roster names an address the resolver listens on, else the roster's first.
|
||||
func Question(listen []string, names []Named) (Named, bool) {
|
||||
for _, n := range names {
|
||||
for _, a := range listen {
|
||||
if n.Address == a {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return names[0], true
|
||||
}
|
||||
return Named{}, false
|
||||
}
|
||||
|
||||
// Look is one look: every listen address asked over both transports.
|
||||
type Look struct {
|
||||
At time.Time `json:"at"`
|
||||
// Healthy says every question was answered rightly.
|
||||
Healthy bool `json:"healthy"`
|
||||
Asked Named `json:"asked"`
|
||||
Answers []Answer `json:"answers"`
|
||||
// Wrong is every question not answered rightly, in words; Problems what kept the look from asking.
|
||||
Wrong []string `json:"wrong,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// Why is what is wrong in one sentence: empty when healthy.
|
||||
func (l Look) Why() string {
|
||||
if l.Healthy {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ")
|
||||
}
|
||||
|
||||
// Look asks every listen address its question, over UDP and TCP, all at once.
|
||||
func (m *Machine) Look(ctx context.Context) Look {
|
||||
l := Look{At: m.Now().UTC()}
|
||||
config, err := m.read(ConfigFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err))
|
||||
return l
|
||||
}
|
||||
listen := ListenAddresses(config)
|
||||
if len(listen) == 0 {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile))
|
||||
return l
|
||||
}
|
||||
nodes, err := m.read(NodesFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err))
|
||||
return l
|
||||
}
|
||||
q, ok := Question(listen, AnsweredNames(nodes))
|
||||
if !ok {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile))
|
||||
return l
|
||||
}
|
||||
l.Asked = q
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, address := range listen {
|
||||
for _, over := range []string{"udp", "tcp"} {
|
||||
wg.Add(1)
|
||||
go func(address, over string) {
|
||||
defer wg.Done()
|
||||
a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin)
|
||||
mu.Lock()
|
||||
l.Answers = append(l.Answers, a)
|
||||
mu.Unlock()
|
||||
}(address, over)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Slice(l.Answers, func(i, j int) bool {
|
||||
if l.Answers[i].Server != l.Answers[j].Server {
|
||||
return l.Answers[i].Server < l.Answers[j].Server
|
||||
}
|
||||
return l.Answers[i].Over > l.Answers[j].Over
|
||||
})
|
||||
for _, a := range l.Answers {
|
||||
if w := wrongWords(a, q); w != "" {
|
||||
l.Wrong = append(l.Wrong, w)
|
||||
}
|
||||
}
|
||||
l.Healthy = len(l.Wrong) == 0
|
||||
return l
|
||||
}
|
||||
|
||||
// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the
|
||||
// machine's address.
|
||||
func wrongWords(a Answer, q Named) string {
|
||||
at := a.Server + " over " + a.Over
|
||||
switch {
|
||||
case !a.Answered:
|
||||
return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed)
|
||||
case a.Code != "NOERROR":
|
||||
return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code)
|
||||
case a.Failed != "":
|
||||
return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed)
|
||||
}
|
||||
for _, x := range a.Addresses {
|
||||
if x == q.Address {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address)
|
||||
}
|
||||
|
||||
func orNothing(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "no address"
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// The dnsmasq module's Go bundle: one of the mesh's resolvers (novox/hq ADR 0194, 0223).
|
||||
//
|
||||
// Started with no arguments it is served by the node's runtime as the operator account, over MCP on stdio
|
||||
// (ADR 0188, ADR 0193): the module's tools and its health, and beside them the events the resolver's
|
||||
// answers give — a machine's name it now answers or no longer does, and a restart the watch made. Started
|
||||
// as `dnsmasq-tools watch` it is the module's long-running process, as root: it asks the resolver a real
|
||||
// question on every listen address and restarts it, within a budget, when it stops answering (watch.go,
|
||||
// novox/hq issue 348). stdout is the MCP channel; everything else is said on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 {
|
||||
if os.Args[1] != "watch" || len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: dnsmasq-tools [watch]")
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
m := Here()
|
||||
if err := m.Watch(ctx, ExecRunner, m.WriteState); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
m := Here()
|
||||
go Announce(m, stdio.Emit)
|
||||
if err := stdio.Serve("", Tools(m)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// AnnounceEvery is how often the bundle reads the roster and the watch for what to say.
|
||||
var AnnounceEvery = 30 * time.Second
|
||||
|
||||
// Announcer remembers what was said, primed silently so a restart of the bundle does not say every name
|
||||
// and every restart again.
|
||||
type Announcer struct {
|
||||
names map[string]string
|
||||
restarts map[time.Time]bool
|
||||
// primed says the names were read once; heard that the watch's restarts were.
|
||||
primed, heard bool
|
||||
}
|
||||
|
||||
// Event is one event to emit.
|
||||
type Event struct {
|
||||
Type string
|
||||
Body map[string]any
|
||||
}
|
||||
|
||||
// Next is what changed since the last read: names the resolver now answers (`name.added`) or no longer
|
||||
// does (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own
|
||||
// events — and restarts the watch made (`restarted`), each with why and whether it helped once known.
|
||||
//
|
||||
// read says the roster was read: an unreadable roster says nothing about names, rather than every name
|
||||
// removed.
|
||||
func (a *Announcer) Next(names []Named, read bool, s State) []Event {
|
||||
now := map[string]string{}
|
||||
for _, n := range names {
|
||||
now[n.Name] = n.Address
|
||||
}
|
||||
var out []Event
|
||||
if a.primed && read {
|
||||
for name, address := range now {
|
||||
if _, had := a.names[name]; !had {
|
||||
out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}})
|
||||
}
|
||||
}
|
||||
for name := range a.names {
|
||||
if _, has := now[name]; !has {
|
||||
out = append(out, Event{"name.removed", map[string]any{"name": name}})
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.restarts == nil {
|
||||
a.restarts = map[time.Time]bool{}
|
||||
}
|
||||
for _, r := range s.Restarts {
|
||||
if a.restarts[r.At] {
|
||||
continue
|
||||
}
|
||||
a.restarts[r.At] = true
|
||||
if a.heard {
|
||||
body := map[string]any{"unit": Unit, "at": r.At, "why": r.Why}
|
||||
if r.Error != "" {
|
||||
body["error"] = r.Error
|
||||
}
|
||||
out = append(out, Event{"restarted", body})
|
||||
}
|
||||
}
|
||||
a.heard = true
|
||||
if read {
|
||||
a.names, a.primed = now, true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Announce says the events for as long as the bundle runs.
|
||||
func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) {
|
||||
time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs
|
||||
var a Announcer
|
||||
for {
|
||||
var names []Named
|
||||
text, err := m.read(NodesFile)
|
||||
if err == nil {
|
||||
names = AnsweredNames(text)
|
||||
}
|
||||
s, _, _ := m.ReadState()
|
||||
for _, e := range a.Next(names, err == nil, s) {
|
||||
if err := emit(e.Type, e.Body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq: %s not said: %v\n", e.Type, err)
|
||||
}
|
||||
}
|
||||
time.Sleep(AnnounceEvery)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Stale is how old the watch's last look may be before the health tool says it stopped looking.
|
||||
const Stale = 2 * time.Minute
|
||||
|
||||
// Health is the module's health answer (novox/hq ADR 0240): healthy only when the resolver answers its
|
||||
// real question on every listen address, over UDP and TCP, and the watch is looking.
|
||||
type Health struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Look Look `json:"look"`
|
||||
// Watch is what the watch last saw and did; absent when it has written nothing yet.
|
||||
Watch *State `json:"watch,omitempty"`
|
||||
}
|
||||
|
||||
// Health asks the resolver now, and reads what the watch did.
|
||||
func (m *Machine) Health(ctx context.Context) Health {
|
||||
l := m.Look(ctx)
|
||||
h := Health{Healthy: l.Healthy, Look: l}
|
||||
var why []string
|
||||
if !l.Healthy {
|
||||
why = append(why, "the resolver does not answer as it must: "+l.Why())
|
||||
}
|
||||
s, had, err := m.ReadState()
|
||||
switch {
|
||||
case err != nil:
|
||||
h.Healthy = false
|
||||
why = append(why, "what the watch saw cannot be read: "+err.Error())
|
||||
case !had:
|
||||
h.Healthy = false
|
||||
why = append(why, "the watch has written nothing yet, so nothing restarts the resolver when it stops answering")
|
||||
default:
|
||||
h.Watch = &s
|
||||
if age := m.Now().Sub(s.Last.At); age > Stale {
|
||||
h.Healthy = false
|
||||
why = append(why, fmt.Sprintf("the watch last looked %s ago, so nothing restarts the resolver when it "+
|
||||
"stops answering", age.Round(time.Second)))
|
||||
}
|
||||
if !l.Healthy {
|
||||
why = append(why, restartWords(s))
|
||||
}
|
||||
}
|
||||
h.Why = strings.Join(nonEmpty(why), "; ")
|
||||
return h
|
||||
}
|
||||
|
||||
// restartWords is what the watch did about it, in words.
|
||||
func restartWords(s State) string {
|
||||
if len(s.Restarts) == 0 {
|
||||
return "the watch has not restarted it"
|
||||
}
|
||||
r := s.Restarts[len(s.Restarts)-1]
|
||||
words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC"))
|
||||
switch {
|
||||
case r.Error != "":
|
||||
words += ", and the restart failed: " + r.Error
|
||||
case r.Helped != nil && !*r.Helped:
|
||||
words += ", which did not help"
|
||||
case r.Helped == nil:
|
||||
words += ", and its next looks decide whether that helped"
|
||||
}
|
||||
if s.Held != "" {
|
||||
words += "; " + s.Held
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
func nonEmpty(s []string) []string {
|
||||
var out []string
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func arg(a map[string]any, k string) string {
|
||||
v, _ := a[k].(string)
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
// Tools are the module's tools.
|
||||
func Tools(m *Machine) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "dnsmasq_health", Description: "Whether this machine's mesh resolver answers: its real question — this " +
|
||||
"machine's own name — asked now on every address its configuration listens on, over UDP and TCP, each " +
|
||||
"answer with its code, addresses and time; and what the module's watch last saw, the restarts it made " +
|
||||
"within the hour, whether each helped, and why it holds back from another. Healthy only when every " +
|
||||
"question was answered rightly and the watch is looking (novox/hq ADR 0240, issue 348). (r)",
|
||||
Input: map[string]any{},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return m.Health(context.Background()), nil
|
||||
}},
|
||||
{Name: "dnsmasq_names", Description: "The mesh's machine names this machine's resolver answers, each with its " +
|
||||
"private address, as the roster wrote them (" + NodesFile + "). (r)",
|
||||
Input: map[string]any{},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
text, err := m.read(NodesFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the roster %s cannot be read: %w", NodesFile, err)
|
||||
}
|
||||
return map[string]any{"names": AnsweredNames(text)}, nil
|
||||
}},
|
||||
{Name: "dnsmasq_resolve", Description: "Ask this machine's resolver one name's IPv4 address, directly, on one " +
|
||||
"address it listens on (default every one) over udp or tcp (default both): the answer code, the addresses " +
|
||||
"and the time each took, or why none came — refused when nothing listens there. Replaces dig @address. (r)",
|
||||
Input: map[string]any{"name": str("the name to ask, e.g. plex.anchor.internal"),
|
||||
"address": str("one listen address to ask (default: every one in its configuration)"),
|
||||
"over": str("udp or tcp (default: both)")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return m.Resolve(context.Background(), arg(a, "name"), arg(a, "address"), arg(a, "over"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve asks name on one listen address or every one, over one transport or both.
|
||||
func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, error) {
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
var addresses []string
|
||||
if address != "" {
|
||||
addresses = []string{address}
|
||||
} else {
|
||||
config, err := m.read(ConfigFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err)
|
||||
}
|
||||
addresses = ListenAddresses(config)
|
||||
}
|
||||
overs := []string{"udp", "tcp"}
|
||||
switch over {
|
||||
case "":
|
||||
case "udp", "tcp":
|
||||
overs = []string{over}
|
||||
default:
|
||||
return nil, fmt.Errorf("over is udp or tcp, not %q", over)
|
||||
}
|
||||
var answers []Answer
|
||||
for _, a := range addresses {
|
||||
for _, o := range overs {
|
||||
answers = append(answers, m.Ask(ctx, a+":"+m.Port, o, name, AskWithin))
|
||||
}
|
||||
}
|
||||
return map[string]any{"name": name, "answers": answers}, nil
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real
|
||||
// question on every listen address (look.go); after Confirm failing looks in a row it restarts the
|
||||
// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh
|
||||
// (novox/hq issue 348).
|
||||
//
|
||||
// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that
|
||||
// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor
|
||||
// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where
|
||||
// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked,
|
||||
// and it never says it worked: its next look does. A restart that did not help is said by the health
|
||||
// tool as the resolver's failure, which the node-engine states and the controller raises — the operator
|
||||
// hears of it only then.
|
||||
|
||||
// The watch's clocks and its budget.
|
||||
var (
|
||||
LookEvery = 15 * time.Second
|
||||
// Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277).
|
||||
Confirm = 2
|
||||
// RestartGap is the least time between two restarts; RestartsPerHour the most within an hour.
|
||||
RestartGap = 2 * time.Minute
|
||||
RestartsPerHour = 3
|
||||
// HelpedWithin is how soon after a restart a healthy look says the restart helped.
|
||||
HelpedWithin = time.Minute
|
||||
)
|
||||
|
||||
// Unit is the resolver's service.
|
||||
const Unit = "dnsmasq.service"
|
||||
|
||||
// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator
|
||||
// account the tools run as, written only by the watch.
|
||||
const StateFile = "/var/lib/dnsmasq-watch/watch.json"
|
||||
|
||||
// Restart is one restart the watch made.
|
||||
type Restart struct {
|
||||
At time.Time `json:"at"`
|
||||
Why string `json:"why"`
|
||||
// Error is what the service manager said when the restart itself failed.
|
||||
Error string `json:"error,omitempty"`
|
||||
// Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once
|
||||
// HelpedWithin passed without one, absent until then.
|
||||
Helped *bool `json:"helped,omitempty"`
|
||||
}
|
||||
|
||||
// State is the watch's last look and what it did, as the tools read it.
|
||||
type State struct {
|
||||
Last Look `json:"last"`
|
||||
// Failing is how many looks in a row failed; Since when the first of them looked.
|
||||
Failing int `json:"failing"`
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
// Restarts are the restarts within the last hour and the last one ever, newest last.
|
||||
Restarts []Restart `json:"restarts,omitempty"`
|
||||
// Held is why a restart that was due was not made: the budget is spent.
|
||||
Held string `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// Runner runs one command; injected, so the watch is tested over recorded answers.
|
||||
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||
|
||||
// ExecRunner runs a command on the machine, bounded to a minute.
|
||||
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
var out bytes.Buffer
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Stdout, cmd.Stderr = &out, &out
|
||||
err := cmd.Run()
|
||||
return strings.TrimSpace(out.String()), err
|
||||
}
|
||||
|
||||
// Step is one look of the watch and what follows from it: the state after, and whether to restart and why.
|
||||
// Pure but for the look it is given.
|
||||
func Step(s State, l Look) (State, string) {
|
||||
now := l.At
|
||||
s.Last = l
|
||||
// What the earlier restarts are now known to have done.
|
||||
for i := range s.Restarts {
|
||||
r := &s.Restarts[i]
|
||||
if r.Helped != nil || r.Error != "" {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case l.Healthy && now.Sub(r.At) <= HelpedWithin:
|
||||
yes := true
|
||||
r.Helped = &yes
|
||||
case now.Sub(r.At) > HelpedWithin:
|
||||
no := false
|
||||
r.Helped = &no
|
||||
}
|
||||
}
|
||||
// The budget's window: the last hour, and always the newest.
|
||||
kept := s.Restarts[:0]
|
||||
for i, r := range s.Restarts {
|
||||
if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
s.Restarts = kept
|
||||
if l.Healthy {
|
||||
s.Failing, s.Since, s.Held = 0, nil, ""
|
||||
return s, ""
|
||||
}
|
||||
if s.Failing == 0 {
|
||||
at := now
|
||||
s.Since = &at
|
||||
}
|
||||
s.Failing++
|
||||
if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 {
|
||||
// Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing
|
||||
// configuration or roster is missing.
|
||||
return s, ""
|
||||
}
|
||||
within := 0
|
||||
var newest *Restart
|
||||
for i, r := range s.Restarts {
|
||||
if now.Sub(r.At) <= time.Hour {
|
||||
within++
|
||||
}
|
||||
newest = &s.Restarts[i]
|
||||
}
|
||||
switch {
|
||||
case newest != nil && now.Sub(newest.At) < RestartGap:
|
||||
s.Held = ""
|
||||
return s, "" // the last restart is still settling
|
||||
case within >= RestartsPerHour:
|
||||
s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+
|
||||
"more until the oldest is an hour old, and the resolver's failure is said as its health", within)
|
||||
return s, ""
|
||||
}
|
||||
s.Held = ""
|
||||
return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why())
|
||||
}
|
||||
|
||||
// Watch runs the watch until ctx ends.
|
||||
func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error {
|
||||
var s State
|
||||
if raw, err := m.read(StateFile); err == nil {
|
||||
_ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart
|
||||
}
|
||||
tick := time.NewTicker(LookEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
l := m.Look(ctx)
|
||||
next, why := Step(s, l)
|
||||
if why != "" {
|
||||
r := Restart{At: m.Now().UTC(), Why: why}
|
||||
if out, err := run(ctx, "systemctl", "restart", Unit); err != nil {
|
||||
r.Error = strings.TrimSpace(err.Error() + " " + out)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error))
|
||||
next.Restarts = append(next.Restarts, r)
|
||||
} else if !l.Healthy && next.Failing == Confirm {
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held))
|
||||
}
|
||||
s = next
|
||||
if err := write(s); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orEmpty(prefix, s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return prefix + s
|
||||
}
|
||||
|
||||
// WriteState writes the state whole, replacing the file in one rename.
|
||||
func (m *Machine) WriteState(s State) error {
|
||||
path := filepath.Join(m.Root, StateFile)
|
||||
body, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, body, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// ReadState is the watch's last state, and false when it has written none.
|
||||
func (m *Machine) ReadState() (State, bool, error) {
|
||||
raw, err := m.read(StateFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return State{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return State{}, false, err
|
||||
}
|
||||
var s State
|
||||
if err := json.Unmarshal([]byte(raw), &s); err != nil {
|
||||
return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err)
|
||||
}
|
||||
return s, true, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module dnsmasq
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
@@ -1,38 +0,0 @@
|
||||
// dnsmasq's events. The resolver's answered set changes whenever the mesh rewrites the wildcard file
|
||||
// and restarts it — a machine joined the private network or left it. This watches that file and
|
||||
// announces, from the resolver's own vantage, that a name became (or stopped being) resolvable on
|
||||
// this node: DNS having actually propagated here, distinct from the mesh's own node.* events.
|
||||
//
|
||||
// Emits (novox/hq ADR 0041/0042):
|
||||
// module.dnsmasq.name.added — this node's resolver now answers a machine's name
|
||||
// module.dnsmasq.name.removed — it no longer does
|
||||
|
||||
import { emit } from "@novox/mesh-sdk/events";
|
||||
import { DnsmasqClient } from "./client.js";
|
||||
|
||||
const dnsmasq = DnsmasqClient.fromEnv();
|
||||
|
||||
// name -> address, primed silently so a restart does not re-announce every name it already answered.
|
||||
const known = new Map<string, string>();
|
||||
let primed = false;
|
||||
|
||||
async function poll(): Promise<void> {
|
||||
const now = new Map((await dnsmasq.answeredNames()).map((a) => [a.name, a.address]));
|
||||
if (primed) {
|
||||
for (const [name, address] of now) {
|
||||
if (!known.has(name)) await emit("name.added", { name, address });
|
||||
}
|
||||
for (const [name] of known) {
|
||||
if (!now.has(name)) await emit("name.removed", { name });
|
||||
}
|
||||
}
|
||||
known.clear();
|
||||
for (const [name, address] of now) known.set(name, address);
|
||||
primed = true;
|
||||
}
|
||||
|
||||
const run = (): void => void poll().catch((err) => console.error(`[dnsmasq] ${err}`));
|
||||
setInterval(run, 30_000);
|
||||
run();
|
||||
|
||||
console.log("[dnsmasq] watching the resolver's answered names");
|
||||
@@ -17,7 +17,8 @@
|
||||
],
|
||||
"emits": [
|
||||
"name.added",
|
||||
"name.removed"
|
||||
"name.removed",
|
||||
"restarted"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker"
|
||||
@@ -28,6 +29,11 @@
|
||||
"scope": "mesh"
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"dnsmasq_health",
|
||||
"dnsmasq_names",
|
||||
"dnsmasq_resolve"
|
||||
],
|
||||
"listens": [
|
||||
{
|
||||
"name": "dns-udp",
|
||||
@@ -75,7 +81,34 @@
|
||||
"config",
|
||||
"dnsmasq.fact-node-zones",
|
||||
"dnsmasq.fact-zones"
|
||||
]
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "watch-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/dnsmasq-watch",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "watch",
|
||||
"type": "process",
|
||||
"name": "dnsmasq-watch",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./dnsmasq-tools",
|
||||
"watch"
|
||||
],
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "dnsmasq_health",
|
||||
"interval": "20s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
}
|
||||
}
|
||||
],
|
||||
"facts": {
|
||||
@@ -87,5 +120,20 @@
|
||||
"path": "/etc/mesh-resolver/zones.conf",
|
||||
"template": "# Generated by the mesh. Do not edit — this file is replaced whenever a module declares a zone or\n# its machine moves (novox/hq ADR 0199). One forwarding line per zone, to the module answering it.\n\n{{range .Zones}}server=/{{.Zone}}/{{.Address}}#{{.Port}}\n{{end}}"
|
||||
}
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/dnsmasq-tools",
|
||||
"binary": "dnsmasq-tools",
|
||||
"loads": [
|
||||
"dnsmasq-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@novox/module-dnsmasq",
|
||||
"version": "0.1.0",
|
||||
"description": "dnsmasq — the mesh's resolver: answers wildcard node names and forwards the rest upstream. Its tools and events live here.",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": { "@novox/mesh-sdk": "^0.1.0" },
|
||||
"devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" }
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
// dnsmasq's tools — a resolver's two useful questions: what does it answer, and does it answer a
|
||||
// given name. Moved into the module (novox/hq ADR 0039); the mesh serves them through the sdk.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { DnsmasqClient } from "../client.js";
|
||||
|
||||
export function getDnsmasqTools(dnsmasq: DnsmasqClient): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "dnsmasq_names",
|
||||
description: "The mesh names this node's resolver answers — each a machine and everything under it.",
|
||||
input: {},
|
||||
run: async () => ({ names: await dnsmasq.answeredNames() }),
|
||||
},
|
||||
{
|
||||
name: "dnsmasq_resolve",
|
||||
description: "Resolve a mesh name through this node's own resolver — the check that wildcard-resolution answers.",
|
||||
input: { name: { type: "string", description: "a name to resolve, e.g. plex.anchor.internal" } },
|
||||
run: async (args) => {
|
||||
const name = String(args.name);
|
||||
try {
|
||||
return { name, addresses: await dnsmasq.resolve(name) };
|
||||
} catch (err) {
|
||||
return { name, addresses: [], error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
registerModuleTools("dnsmasq", (env) => getDnsmasqTools(DnsmasqClient.fromEnv(env)));
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["client.ts", "index.ts", "tools/index.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user