Merge pull request 'dnsmasq answers or is restarted, binds once; tools for DNS, sockets and the runtime's memory (hq issue 348)' (#161) from fix/the-resolver-answers-or-is-restarted into main
This commit was merged in pull request #161.
This commit is contained in:
@@ -7,9 +7,11 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
|
||||
modules/messenger/cmd/messenger/messenger
|
||||
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
|
||||
modules/claude-code/cmd/claude-code/claude-code
|
||||
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
|
||||
modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery
|
||||
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
|
||||
modules/messenger/messenger
|
||||
modules/claude-code/claude-code
|
||||
modules/mesh-watcher/mesh-watcher
|
||||
modules/dnsmasq/dnsmasq
|
||||
modules/mesh-delivery/mesh-delivery
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
70
|
||||
69
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
// dnsmasq's own code, living in the module (novox/hq ADR 0039). dnsmasq here answers the mesh's
|
||||
// generated wildcard names (<service>.<node>.<suffix>) itself and forwards everything else to fixed
|
||||
// upstreams — the predecessor's arrangement, so the machine and its containers ask it for the world.
|
||||
// Its code reads what it was told to answer, and can resolve through itself to prove that it does.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Resolver } from "node:dns/promises";
|
||||
|
||||
export interface AnsweredName {
|
||||
/** A machine's internal name, e.g. "anchor.internal" — it and everything under it resolve here. */
|
||||
name: string;
|
||||
address: string;
|
||||
}
|
||||
|
||||
export class DnsmasqClient {
|
||||
constructor(
|
||||
private readonly resolverPath: string,
|
||||
private readonly address: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Build from the environment. Both values are node-local facts with mesh-chosen defaults — the
|
||||
* wildcard file the module is sent, and the loopback address its config listens on (127.0.0.1,
|
||||
* where the predecessor's resolver answered and every machine's resolv.conf already points) — so
|
||||
* there is nothing to be unconfigured about; it never throws.
|
||||
*/
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): DnsmasqClient {
|
||||
return new DnsmasqClient(
|
||||
env.MESH_DNSMASQ_RESOLVER_PATH ?? "/etc/mesh-resolver/nodes.conf",
|
||||
env.MESH_DNSMASQ_ADDRESS ?? "127.0.0.1",
|
||||
);
|
||||
}
|
||||
|
||||
/** The names this resolver answers, read from the mesh-generated wildcard file. */
|
||||
async answeredNames(): Promise<AnsweredName[]> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(this.resolverPath, "utf8");
|
||||
} catch {
|
||||
return []; // not yet on the network, or the file has not been written — no names, not an error
|
||||
}
|
||||
const names: AnsweredName[] = [];
|
||||
for (const line of text.split("\n")) {
|
||||
// dnsmasq wildcard syntax the mesh writes: address=/<node>.<suffix>/<address>. The file also
|
||||
// carries local=/<suffix>/, which names no machine and is not matched here.
|
||||
const match = line.match(/^address=\/([^/]+)\/(.+)$/);
|
||||
if (match) names.push({ name: match[1], address: match[2] });
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** Resolve a name through this node's own resolver — the check that wildcard-resolution answers. */
|
||||
async resolve(name: string): Promise<string[]> {
|
||||
const resolver = new Resolver();
|
||||
resolver.setServers([this.address]);
|
||||
try {
|
||||
return await resolver.resolve4(name);
|
||||
} catch {
|
||||
return await resolver.resolve6(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A real question to the resolver, as any program on the machine asks it: one name, one record type, over
|
||||
// UDP or TCP, to one address. Written out here rather than taken from a library: the module needs the
|
||||
// header's answer code and the A records, nothing else, and Go's own resolver would read /etc/hosts and
|
||||
// /etc/resolv.conf instead of asking the one address it is told (novox/hq issue 348).
|
||||
|
||||
// Answer is what one question got.
|
||||
type Answer struct {
|
||||
// Server is where it was asked, and Over the transport.
|
||||
Server string `json:"server"`
|
||||
Over string `json:"over"`
|
||||
Name string `json:"name"`
|
||||
// Answered says a reply came; Code is its answer code (NOERROR, NXDOMAIN, …) and Addresses its A
|
||||
// records.
|
||||
Answered bool `json:"answered"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Addresses []string `json:"addresses,omitempty"`
|
||||
// Failed is why no reply came, in words: refused (nothing listens there), no answer within, or what
|
||||
// the system said.
|
||||
Failed string `json:"failed,omitempty"`
|
||||
// Took is how long it took, in milliseconds.
|
||||
Took int64 `json:"took_ms"`
|
||||
}
|
||||
|
||||
// rcodes are the answer codes by number.
|
||||
var rcodes = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
|
||||
|
||||
// query is one question for name's A record, with id.
|
||||
func query(name string, id uint16) ([]byte, error) {
|
||||
name = strings.TrimSuffix(name, ".")
|
||||
if name == "" {
|
||||
return nil, errors.New("no name to ask")
|
||||
}
|
||||
msg := make([]byte, 12, 64)
|
||||
binary.BigEndian.PutUint16(msg[0:], id)
|
||||
binary.BigEndian.PutUint16(msg[2:], 0x0100) // a standard query, recursion desired
|
||||
binary.BigEndian.PutUint16(msg[4:], 1) // one question
|
||||
for _, label := range strings.Split(name, ".") {
|
||||
if label == "" || len(label) > 63 {
|
||||
return nil, fmt.Errorf("%q is not a name that can be asked", name)
|
||||
}
|
||||
msg = append(msg, byte(len(label)))
|
||||
msg = append(msg, label...)
|
||||
}
|
||||
msg = append(msg, 0, 0, 1, 0, 1) // root; type A; class IN
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// readAnswer reads a reply to the question with id: its code and A records.
|
||||
func readAnswer(msg []byte, id uint16) (string, []string, error) {
|
||||
if len(msg) < 12 {
|
||||
return "", nil, errors.New("a reply too short to be one")
|
||||
}
|
||||
if binary.BigEndian.Uint16(msg[0:]) != id {
|
||||
return "", nil, errors.New("a reply to another question")
|
||||
}
|
||||
flags := binary.BigEndian.Uint16(msg[2:])
|
||||
code, ok := rcodes[int(flags&0xF)]
|
||||
if !ok {
|
||||
code = fmt.Sprintf("RCODE%d", flags&0xF)
|
||||
}
|
||||
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
|
||||
at := 12
|
||||
for i := 0; i < qd; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return code, nil, err
|
||||
}
|
||||
at += 4
|
||||
}
|
||||
var addresses []string
|
||||
for i := 0; i < an; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return code, addresses, err
|
||||
}
|
||||
if at+10 > len(msg) {
|
||||
return code, addresses, errors.New("a reply cut short")
|
||||
}
|
||||
kind, length := binary.BigEndian.Uint16(msg[at:]), int(binary.BigEndian.Uint16(msg[at+8:]))
|
||||
at += 10
|
||||
if at+length > len(msg) {
|
||||
return code, addresses, errors.New("a reply cut short")
|
||||
}
|
||||
if kind == 1 && length == 4 {
|
||||
addresses = append(addresses, net.IP(msg[at:at+4]).String())
|
||||
}
|
||||
at += length
|
||||
}
|
||||
return code, addresses, nil
|
||||
}
|
||||
|
||||
// skipName steps over a name in a message, a pointer ending it.
|
||||
func skipName(msg []byte, at int) (int, error) {
|
||||
for at < len(msg) {
|
||||
n := int(msg[at])
|
||||
switch {
|
||||
case n == 0:
|
||||
return at + 1, nil
|
||||
case n&0xC0 == 0xC0:
|
||||
return at + 2, nil
|
||||
}
|
||||
at += n + 1
|
||||
}
|
||||
return at, errors.New("a name runs past the reply")
|
||||
}
|
||||
|
||||
// Ask asks server (an address, port 53 unless given) for name's A record over "udp" or "tcp", within
|
||||
// timeout.
|
||||
func Ask(ctx context.Context, server, over, name string, timeout time.Duration) Answer {
|
||||
if _, _, err := net.SplitHostPort(server); err != nil {
|
||||
server = net.JoinHostPort(server, "53")
|
||||
}
|
||||
a := Answer{Server: server, Over: over, Name: name}
|
||||
started := time.Now()
|
||||
defer func() { a.Took = time.Since(started).Milliseconds() }()
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
id := uint16(rand.UintN(1 << 16))
|
||||
q, err := query(name, id)
|
||||
if err != nil {
|
||||
a.Failed = err.Error()
|
||||
return a
|
||||
}
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, over, server)
|
||||
if err != nil {
|
||||
a.Failed = failedWords(err, timeout)
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
defer conn.Close()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = conn.SetDeadline(deadline)
|
||||
}
|
||||
var reply []byte
|
||||
if over == "tcp" {
|
||||
framed := binary.BigEndian.AppendUint16(nil, uint16(len(q)))
|
||||
if _, err = conn.Write(append(framed, q...)); err == nil {
|
||||
head := make([]byte, 2)
|
||||
if _, err = readFull(conn, head); err == nil {
|
||||
reply = make([]byte, binary.BigEndian.Uint16(head))
|
||||
_, err = readFull(conn, reply)
|
||||
}
|
||||
}
|
||||
} else if _, err = conn.Write(q); err == nil {
|
||||
buf := make([]byte, 1500)
|
||||
var n int
|
||||
n, err = conn.Read(buf)
|
||||
reply = buf[:n]
|
||||
}
|
||||
if err != nil {
|
||||
a.Failed = failedWords(err, timeout)
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
a.Answered = true
|
||||
a.Code, a.Addresses, err = readAnswer(reply, id)
|
||||
if err != nil {
|
||||
a.Failed = err.Error()
|
||||
}
|
||||
a.Took = time.Since(started).Milliseconds()
|
||||
return a
|
||||
}
|
||||
|
||||
func readFull(conn net.Conn, buf []byte) (int, error) {
|
||||
read := 0
|
||||
for read < len(buf) {
|
||||
n, err := conn.Read(buf[read:])
|
||||
read += n
|
||||
if err != nil {
|
||||
return read, err
|
||||
}
|
||||
}
|
||||
return read, nil
|
||||
}
|
||||
|
||||
// failedWords is why no reply came: refused when nothing listens, no answer when nothing came in time.
|
||||
func failedWords(err error, timeout time.Duration) string {
|
||||
var timedOut interface{ Timeout() bool }
|
||||
switch {
|
||||
case errors.Is(err, syscall.ECONNREFUSED):
|
||||
return "refused: nothing listens there"
|
||||
case errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timedOut) && timedOut.Timeout()):
|
||||
return fmt.Sprintf("no answer within %s", timeout)
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
@@ -0,0 +1,559 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
|
||||
// stopped — as dnsmasq answers the machine's own name from the roster.
|
||||
type standIn struct {
|
||||
udp net.PacketConn
|
||||
tcp net.Listener
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
func reply(q []byte, address string) []byte {
|
||||
// The question back, with the answer: header flags QR+RD+RA, one answer, a pointer to the name.
|
||||
out := append([]byte{}, q...)
|
||||
binary.BigEndian.PutUint16(out[2:], 0x8180)
|
||||
binary.BigEndian.PutUint16(out[6:], 1)
|
||||
ip := net.ParseIP(address).To4()
|
||||
out = append(out, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4)
|
||||
return append(out, ip...)
|
||||
}
|
||||
|
||||
func startStandIn(t *testing.T, at, port, address string) *standIn {
|
||||
t.Helper()
|
||||
s := &standIn{}
|
||||
var err error
|
||||
if s.udp, err = net.ListenPacket("udp", net.JoinHostPort(at, port)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.tcp, err = net.Listen("tcp", net.JoinHostPort(at, port)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.wg.Add(2)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, from, err := s.udp.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = s.udp.WriteTo(reply(buf[:n], address), from)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
c, err := s.tcp.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
head := make([]byte, 2)
|
||||
if _, err := readFull(c, head); err == nil {
|
||||
q := make([]byte, binary.BigEndian.Uint16(head))
|
||||
if _, err := readFull(c, q); err == nil {
|
||||
r := reply(q, address)
|
||||
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
|
||||
}
|
||||
}
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
t.Cleanup(s.stop)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *standIn) stop() {
|
||||
_ = s.udp.Close()
|
||||
_ = s.tcp.Close()
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
// freePort is a port free on both loopback addresses the tests use.
|
||||
func freePort(t *testing.T) string {
|
||||
t.Helper()
|
||||
for i := 0; i < 20; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
|
||||
_ = l.Close()
|
||||
ok := true
|
||||
for _, at := range []string{"127.0.0.1", "127.0.0.2"} {
|
||||
for _, over := range []string{"tcp", "udp"} {
|
||||
if over == "tcp" {
|
||||
if c, err := net.Listen(over, net.JoinHostPort(at, port)); err == nil {
|
||||
_ = c.Close()
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
} else if c, err := net.ListenPacket(over, net.JoinHostPort(at, port)); err == nil {
|
||||
_ = c.Close()
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return port
|
||||
}
|
||||
}
|
||||
t.Fatal("no port free on both loopback addresses")
|
||||
return ""
|
||||
}
|
||||
|
||||
// aResolverMachine is a machine whose resolver listens on 127.0.0.2 (standing for its private address)
|
||||
// and 127.0.0.1, as the module's configuration says, and whose roster names it at 127.0.0.2.
|
||||
func aResolverMachine(t *testing.T, port string, now func() time.Time) *Machine {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
write := func(path, text string) {
|
||||
p := filepath.Join(root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(text), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(ConfigFile, "bind-interfaces\nlisten-address=127.0.0.2\nlisten-address=127.0.0.1\nno-resolv\n")
|
||||
write(NodesFile, "local=/internal/\naddress=/novox.internal/127.0.0.2\nhost-record=novox.internal,127.0.0.2\n"+
|
||||
"address=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n")
|
||||
if err := os.MkdirAll(filepath.Join(root, filepath.Dir(StateFile)), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &Machine{Root: root, Ask: Ask, Now: now, Port: port}
|
||||
}
|
||||
|
||||
func TestTheConfigurationAndTheRosterSayWhereAndWhatToAsk(t *testing.T) {
|
||||
listen := ListenAddresses("bind-dynamic\nlisten-address=10.77.0.1\n# listen-address=9.9.9.9\nlisten-address=127.0.0.1\n")
|
||||
if strings.Join(listen, " ") != "10.77.0.1 127.0.0.1" {
|
||||
t.Fatalf("listen addresses %v", listen)
|
||||
}
|
||||
names := AnsweredNames("local=/internal/\naddress=/ace.internal/10.77.0.2\nhost-record=ace.internal,10.77.0.2\n" +
|
||||
"host-record=novox.internal,10.77.0.1\n")
|
||||
q, ok := Question(listen, names)
|
||||
if !ok || q.Name != "novox.internal" || q.Address != "10.77.0.1" {
|
||||
t.Fatalf("asks %+v", q)
|
||||
}
|
||||
if q, _ := Question([]string{"127.0.0.1"}, names); q.Name != "ace.internal" {
|
||||
t.Fatalf("with no own name in the roster, asks %+v", q)
|
||||
}
|
||||
}
|
||||
|
||||
// **The outage of 2026-10-09** (novox/hq issue 348): the resolver kept running and answering on loopback,
|
||||
// and refused every question on its private address. Its unit said healthy. The look does not.
|
||||
func TestALookFindsAResolverThatStoppedAnsweringOnOneAddress(t *testing.T) {
|
||||
port := freePort(t)
|
||||
m := aResolverMachine(t, port, time.Now)
|
||||
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
if l := m.Look(t.Context()); !l.Healthy || len(l.Answers) != 4 || l.Asked.Name != "novox.internal" {
|
||||
t.Fatalf("a resolver answering everywhere: %+v", l)
|
||||
}
|
||||
private.stop()
|
||||
l := m.Look(t.Context())
|
||||
if l.Healthy || len(l.Wrong) != 2 {
|
||||
t.Fatalf("a resolver that stopped answering on its private address: %+v", l)
|
||||
}
|
||||
if why := l.Why(); !strings.Contains(why, "127.0.0.2:"+port+" over udp did not answer novox.internal: refused") ||
|
||||
!strings.Contains(why, "over tcp") || strings.Contains(why, "127.0.0.1:") {
|
||||
t.Fatalf("said as %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWrongAnswerIsNotAnAnswer(t *testing.T) {
|
||||
q := Named{Name: "novox.internal", Address: "10.77.0.1"}
|
||||
for _, c := range []struct {
|
||||
a Answer
|
||||
want string
|
||||
}{
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"10.77.0.1"}}, ""},
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NXDOMAIN"}, "with NXDOMAIN"},
|
||||
{Answer{Server: "s", Over: "udp", Answered: true, Code: "NOERROR", Addresses: []string{"1.2.3.4"}}, "not 10.77.0.1"},
|
||||
{Answer{Server: "s", Over: "udp", Failed: "no answer within 2s"}, "did not answer"},
|
||||
} {
|
||||
if got := wrongWords(c.a, q); (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
|
||||
t.Errorf("%+v said %q, want %q", c.a, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func failing(at time.Time) Look {
|
||||
return Look{At: at, Answers: []Answer{{Server: "10.77.0.1:53", Over: "udp", Failed: "refused: nothing listens there"}},
|
||||
Wrong: []string{"10.77.0.1:53 over udp did not answer novox.internal: refused: nothing listens there"}}
|
||||
}
|
||||
|
||||
func healthy(at time.Time) Look { return Look{At: at, Healthy: true} }
|
||||
|
||||
// The watch restarts after Confirm failing looks, not one; says whether the restart helped from the looks
|
||||
// after it; waits RestartGap before another; and stops at RestartsPerHour, saying why.
|
||||
func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
var s State
|
||||
var why string
|
||||
if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 {
|
||||
t.Fatalf("one failing look restarted: %q", why)
|
||||
}
|
||||
if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
|
||||
t.Fatalf("a second failing look did not restart: %+v %q", s, why)
|
||||
}
|
||||
s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
|
||||
// It helped.
|
||||
s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active")
|
||||
if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
|
||||
t.Fatalf("a healthy look after a restart: %+v %q", s, why)
|
||||
}
|
||||
// Failing again soon after: held for the gap, then restarted; and so on until the budget is spent.
|
||||
at := t0.Add(3 * LookEvery)
|
||||
restarts := 1
|
||||
for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
|
||||
at = at.Add(LookEvery)
|
||||
s, why = Step(s, failing(at), "active")
|
||||
if why == "" {
|
||||
continue
|
||||
}
|
||||
last := s.Restarts[len(s.Restarts)-1].At
|
||||
if at.Sub(last) < RestartGap {
|
||||
t.Fatalf("restarted %s after the last", at.Sub(last))
|
||||
}
|
||||
s.Restarts = append(s.Restarts, Restart{At: at, Why: why})
|
||||
restarts++
|
||||
}
|
||||
if restarts != RestartsPerHour {
|
||||
t.Fatalf("%d restarts within the hour, want %d", restarts, RestartsPerHour)
|
||||
}
|
||||
if !strings.Contains(s.Held, "the most the watch makes") {
|
||||
t.Fatalf("the spent budget is not said: %+v", s)
|
||||
}
|
||||
for _, r := range s.Restarts[1:] {
|
||||
if r.Helped == nil || *r.Helped {
|
||||
t.Fatalf("a restart followed only by failing looks is said as %v", r.Helped)
|
||||
}
|
||||
}
|
||||
// A look that could not ask — no configuration — restarts nothing.
|
||||
var none State
|
||||
broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
|
||||
none, _ = Step(none, broken, "active")
|
||||
if _, why := Step(none, broken, "active"); why != "" {
|
||||
t.Fatalf("a missing configuration restarted the resolver: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// **The acceptance, end to end on a stand-in**: the resolver stops answering on its private address; the
|
||||
// watch notices on its second look, restarts the unit (which brings the listener back), says the restart
|
||||
// helped, and the health tool answers healthy — the operator hears nothing. And when the restart does not
|
||||
// help, the health tool says so, with the restart, so the node-engine states it and the mesh tells the
|
||||
// operator.
|
||||
func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelped(t *testing.T) {
|
||||
port := freePort(t)
|
||||
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
var mu sync.Mutex
|
||||
now := func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
|
||||
pass := func(d time.Duration) { mu.Lock(); clock = clock.Add(d); mu.Unlock() }
|
||||
m := aResolverMachine(t, port, now)
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
private := startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
private.stop()
|
||||
|
||||
fixes := true
|
||||
var asked []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||
if fixes {
|
||||
private = startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
var s State
|
||||
look := func() {
|
||||
t.Helper()
|
||||
next, why := Step(s, m.Look(t.Context()), "active")
|
||||
if why != "" {
|
||||
r := Restart{At: now().UTC(), Why: why}
|
||||
if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
|
||||
r.Error = err.Error()
|
||||
}
|
||||
next.Restarts = append(next.Restarts, r)
|
||||
}
|
||||
s = next
|
||||
if err := m.WriteState(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pass(LookEvery)
|
||||
}
|
||||
look()
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has not restarted it") {
|
||||
t.Fatalf("after one failing look: %+v", h.Why)
|
||||
}
|
||||
look()
|
||||
if len(asked) != 1 || asked[0] != "systemctl restart dnsmasq.service" {
|
||||
t.Fatalf("restarted with %v", asked)
|
||||
}
|
||||
look()
|
||||
if r := s.Restarts[0]; r.Helped == nil || !*r.Helped {
|
||||
t.Fatalf("the restart that brought it back is said as %+v", r)
|
||||
}
|
||||
if h := m.Health(t.Context()); !h.Healthy || h.Watch == nil || len(h.Watch.Restarts) != 1 {
|
||||
t.Fatalf("after the restart helped: %+v", h)
|
||||
}
|
||||
|
||||
// It stops again, and this time a restart does not help.
|
||||
private.stop()
|
||||
fixes = false
|
||||
pass(RestartGap)
|
||||
for i := 0; i < 2+int(HelpedWithin/LookEvery)+1; i++ {
|
||||
look()
|
||||
}
|
||||
h := m.Health(t.Context())
|
||||
if h.Healthy || !strings.Contains(h.Why, "did not answer novox.internal: refused") ||
|
||||
!strings.Contains(h.Why, "which did not help") {
|
||||
t.Fatalf("a restart that did not help: %q", h.Why)
|
||||
}
|
||||
// And a watch that stopped looking is said, whatever the resolver does.
|
||||
pass(Stale + time.Minute)
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked") {
|
||||
t.Fatalf("a stale watch: %q", h.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came
|
||||
// back on its own, or by something else.
|
||||
func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
s := State{Restarts: []Restart{{At: t0}}}
|
||||
s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active")
|
||||
if h := s.Restarts[0].Helped; h == nil || *h {
|
||||
t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h)
|
||||
}
|
||||
s = State{Restarts: []Restart{{At: t0}}}
|
||||
s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active")
|
||||
if h := s.Restarts[0].Helped; h == nil || !*h {
|
||||
t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h)
|
||||
}
|
||||
}
|
||||
|
||||
// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is
|
||||
// held, said, and not counted against the budget (review of PR 161, C2).
|
||||
func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) {
|
||||
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
for _, unit := range []string{"inactive", "activating", "failed", ""} {
|
||||
var s State
|
||||
var why string
|
||||
for i := 0; i < 4; i++ {
|
||||
s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit)
|
||||
if why != "" {
|
||||
t.Fatalf("a unit %q was restarted", unit)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 {
|
||||
t.Fatalf("a unit %q: %+v", unit, s)
|
||||
}
|
||||
// Running again and still not answering: restarted, the held looks counted as failing.
|
||||
if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" {
|
||||
t.Fatalf("a unit running again after %q, not answering, was not restarted", unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts
|
||||
// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4).
|
||||
func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) {
|
||||
port := freePort(t)
|
||||
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
|
||||
m := aResolverMachine(t, port, func() time.Time { return clock })
|
||||
m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer {
|
||||
return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"}
|
||||
}
|
||||
before := LookEvery
|
||||
LookEvery = time.Millisecond
|
||||
t.Cleanup(func() { LookEvery = before })
|
||||
run := func(spent []Restart) ([]string, State) {
|
||||
t.Helper()
|
||||
if err := m.WriteState(State{Restarts: spent}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var asked []string
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
looks := 0
|
||||
var last State
|
||||
err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) {
|
||||
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "show" {
|
||||
return "active\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}, func(s State) error {
|
||||
last = s
|
||||
if looks++; looks == 3 {
|
||||
cancel()
|
||||
}
|
||||
return m.WriteState(s)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return asked, last
|
||||
}
|
||||
// Three restarts within the hour before it was restarted: it restarts nothing more.
|
||||
spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}}
|
||||
asked, last := run(spent)
|
||||
for _, a := range asked {
|
||||
if strings.Contains(a, "restart") {
|
||||
t.Fatalf("restarted with its budget spent before it started: %v", asked)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 {
|
||||
t.Fatalf("the budget it kept: %+v", last)
|
||||
}
|
||||
// None spent: restarted on its second failing look, not waiting for the start.
|
||||
asked, last = run(nil)
|
||||
restarts := 0
|
||||
for _, a := range asked {
|
||||
if strings.HasPrefix(a, "systemctl restart") {
|
||||
restarts++
|
||||
if a != "systemctl restart --no-block dnsmasq.service" {
|
||||
t.Fatalf("restarted as %q", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" {
|
||||
t.Fatalf("restarts %d: %v %+v", restarts, asked, last)
|
||||
}
|
||||
}
|
||||
|
||||
// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running.
|
||||
func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) {
|
||||
port := freePort(t)
|
||||
clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
|
||||
m := aResolverMachine(t, port, func() time.Time { return clock })
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") {
|
||||
t.Fatalf("no watch: %+v", h.Why)
|
||||
}
|
||||
if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") {
|
||||
t.Fatalf("a stale watch: %+v", h.Why)
|
||||
}
|
||||
if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := m.Health(t.Context()); !h.Healthy || h.Why != "" {
|
||||
t.Fatalf("a watch looking, a resolver answering: %+v", h)
|
||||
}
|
||||
if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") {
|
||||
t.Fatalf("a unit stopped: %+v", h.Why)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") {
|
||||
t.Fatalf("an unreadable state: %+v", h.Why)
|
||||
}
|
||||
}
|
||||
|
||||
// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3).
|
||||
func TestResolveAsksOnlyAListenAddress(t *testing.T) {
|
||||
port := freePort(t)
|
||||
m := aResolverMachine(t, port, time.Now)
|
||||
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
|
||||
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
|
||||
if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") {
|
||||
t.Fatalf("another address was asked: %v", err)
|
||||
}
|
||||
got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answers := got.(map[string]any)["answers"].([]Answer)
|
||||
if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" {
|
||||
t.Fatalf("%+v", answers)
|
||||
}
|
||||
if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 {
|
||||
t.Fatalf("every listen address over both: %+v", got)
|
||||
}
|
||||
if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil {
|
||||
t.Fatal("over quic was accepted")
|
||||
}
|
||||
if _, err := m.Resolve(t.Context(), "", "", ""); err == nil {
|
||||
t.Fatal("no name was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// The announcer: names primed silently, restarts made before the bundle started said, an event remembered
|
||||
// only once emitted, and whether a restart helped said (review of PR 161, C5).
|
||||
func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) {
|
||||
port := freePort(t)
|
||||
t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
|
||||
m := aResolverMachine(t, port, func() time.Time { return t0 })
|
||||
yes := true
|
||||
if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said []string
|
||||
failing := true
|
||||
emit := func(kind string, body any, _ ...stdio.EmitOptions) error {
|
||||
if failing && kind == "restarted" {
|
||||
return errors.New("no bus")
|
||||
}
|
||||
said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"]))
|
||||
return nil
|
||||
}
|
||||
var a Announcer
|
||||
Tell(m, &a, emit)
|
||||
if strings.Join(said, "|") != "restart-judged <nil> true" {
|
||||
t.Fatalf("at the start: %v", said)
|
||||
}
|
||||
failing, said = false, nil
|
||||
Tell(m, &a, emit)
|
||||
if strings.Join(said, "|") != "restarted <nil> <nil>" {
|
||||
t.Fatalf("a restart whose emit failed was not said again: %v", said)
|
||||
}
|
||||
said = nil
|
||||
Tell(m, &a, emit)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said twice: %v", said)
|
||||
}
|
||||
// The roster changes: one machine added, one gone. An unreadable roster says nothing.
|
||||
nodes := filepath.Join(m.Root, NodesFile)
|
||||
if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
Tell(m, &a, emit)
|
||||
if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") ||
|
||||
!strings.Contains(got, "name.removed ace.internal") || len(said) != 2 {
|
||||
t.Fatalf("the roster's change: %v", said)
|
||||
}
|
||||
said = nil
|
||||
if err := os.Remove(nodes); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
Tell(m, &a, emit)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("an unreadable roster said %v", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A look asks the resolver one real question on every address its configuration tells it to answer on,
|
||||
// over UDP and over TCP: the machine's own name, which the mesh's roster gives every holder of the seat
|
||||
// (novox/hq ADR 0194, 0223). On 2026-10-09 the control node's resolver stayed active and running, its
|
||||
// journal silent, and refused every question on its private address for twenty minutes; its unit said
|
||||
// healthy throughout (novox/hq issue 348). A unit that runs is not a resolver that answers.
|
||||
|
||||
// The files a look reads, under the machine's root.
|
||||
const (
|
||||
ConfigFile = "/etc/dnsmasq.conf"
|
||||
NodesFile = "/etc/mesh-resolver/nodes.conf"
|
||||
)
|
||||
|
||||
// AskWithin bounds each question.
|
||||
const AskWithin = 2 * time.Second
|
||||
|
||||
// Machine is a filesystem root (the real one, or a test's tree), how a question is asked, and a clock.
|
||||
type Machine struct {
|
||||
Root string
|
||||
Ask func(ctx context.Context, server, over, name string, timeout time.Duration) Answer
|
||||
Now func() time.Time
|
||||
// Port is where the resolver answers; 53 but in a test.
|
||||
Port string
|
||||
}
|
||||
|
||||
// Here is this machine.
|
||||
func Here() *Machine {
|
||||
return &Machine{Root: "/", Ask: Ask, Now: time.Now, Port: "53"}
|
||||
}
|
||||
|
||||
func (m *Machine) read(path string) (string, error) {
|
||||
b, err := os.ReadFile(filepath.Join(m.Root, path))
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// ListenAddresses are the addresses the resolver's configuration tells it to answer on, in its order.
|
||||
func ListenAddresses(config string) []string {
|
||||
var out []string
|
||||
sc := bufio.NewScanner(strings.NewReader(config))
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
value, ok := strings.CutPrefix(line, "listen-address=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, a := range strings.Split(value, ",") {
|
||||
if a = strings.TrimSpace(a); a != "" {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Named is a machine's name and its private address, as the roster gave them to the resolver.
|
||||
type Named struct {
|
||||
Name string `json:"name"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
// AnsweredNames are the machines' names the roster tells the resolver to answer, each with its address:
|
||||
// its `host-record=<name>,<address>` lines.
|
||||
func AnsweredNames(nodes string) []Named {
|
||||
var out []Named
|
||||
sc := bufio.NewScanner(strings.NewReader(nodes))
|
||||
for sc.Scan() {
|
||||
value, ok := strings.CutPrefix(strings.TrimSpace(sc.Text()), "host-record=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, address, ok := strings.Cut(value, ",")
|
||||
if ok && name != "" && address != "" {
|
||||
out = append(out, Named{Name: name, Address: strings.TrimSpace(address)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Question is the name a look asks, and the address a right answer carries: the machine's own when the
|
||||
// roster names an address the resolver listens on, else the roster's first.
|
||||
func Question(listen []string, names []Named) (Named, bool) {
|
||||
for _, n := range names {
|
||||
for _, a := range listen {
|
||||
if n.Address == a {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(names) > 0 {
|
||||
return names[0], true
|
||||
}
|
||||
return Named{}, false
|
||||
}
|
||||
|
||||
// Look is one look: every listen address asked over both transports.
|
||||
type Look struct {
|
||||
At time.Time `json:"at"`
|
||||
// Healthy says every question was answered rightly.
|
||||
Healthy bool `json:"healthy"`
|
||||
Asked Named `json:"asked"`
|
||||
Answers []Answer `json:"answers"`
|
||||
// Wrong is every question not answered rightly, in words; Problems what kept the look from asking.
|
||||
Wrong []string `json:"wrong,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// Why is what is wrong in one sentence: empty when healthy.
|
||||
func (l Look) Why() string {
|
||||
if l.Healthy {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(append(append([]string{}, l.Problems...), l.Wrong...), "; ")
|
||||
}
|
||||
|
||||
// Look asks every listen address its question, over UDP and TCP, all at once.
|
||||
func (m *Machine) Look(ctx context.Context) Look {
|
||||
l := Look{At: m.Now().UTC()}
|
||||
config, err := m.read(ConfigFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("its configuration %s cannot be read: %v", ConfigFile, err))
|
||||
return l
|
||||
}
|
||||
listen := ListenAddresses(config)
|
||||
if len(listen) == 0 {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("%s names no listen-address, so there is nowhere to ask", ConfigFile))
|
||||
return l
|
||||
}
|
||||
nodes, err := m.read(NodesFile)
|
||||
if err != nil {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s cannot be read: %v", NodesFile, err))
|
||||
return l
|
||||
}
|
||||
q, ok := Question(listen, AnsweredNames(nodes))
|
||||
if !ok {
|
||||
l.Problems = append(l.Problems, fmt.Sprintf("the roster %s names no machine, so there is nothing to ask", NodesFile))
|
||||
return l
|
||||
}
|
||||
l.Asked = q
|
||||
var mu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
for _, address := range listen {
|
||||
for _, over := range []string{"udp", "tcp"} {
|
||||
wg.Add(1)
|
||||
go func(address, over string) {
|
||||
defer wg.Done()
|
||||
a := m.Ask(ctx, address+":"+m.Port, over, q.Name, AskWithin)
|
||||
mu.Lock()
|
||||
l.Answers = append(l.Answers, a)
|
||||
mu.Unlock()
|
||||
}(address, over)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
sort.Slice(l.Answers, func(i, j int) bool {
|
||||
if l.Answers[i].Server != l.Answers[j].Server {
|
||||
return l.Answers[i].Server < l.Answers[j].Server
|
||||
}
|
||||
return l.Answers[i].Over > l.Answers[j].Over
|
||||
})
|
||||
for _, a := range l.Answers {
|
||||
if w := wrongWords(a, q); w != "" {
|
||||
l.Wrong = append(l.Wrong, w)
|
||||
}
|
||||
}
|
||||
l.Healthy = len(l.Wrong) == 0
|
||||
return l
|
||||
}
|
||||
|
||||
// wrongWords is what is wrong with one answer, in words; empty when it is right: NOERROR with the
|
||||
// machine's address.
|
||||
func wrongWords(a Answer, q Named) string {
|
||||
at := a.Server + " over " + a.Over
|
||||
switch {
|
||||
case !a.Answered:
|
||||
return fmt.Sprintf("%s did not answer %s: %s", at, q.Name, a.Failed)
|
||||
case a.Code != "NOERROR":
|
||||
return fmt.Sprintf("%s answered %s with %s", at, q.Name, a.Code)
|
||||
case a.Failed != "":
|
||||
return fmt.Sprintf("%s answered %s with a reply that cannot be read: %s", at, q.Name, a.Failed)
|
||||
}
|
||||
for _, x := range a.Addresses {
|
||||
if x == q.Address {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s answered %s with %s, not %s", at, q.Name, orNothing(a.Addresses), q.Address)
|
||||
}
|
||||
|
||||
func orNothing(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "no address"
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// The dnsmasq module's Go bundle: one of the mesh's resolvers (novox/hq ADR 0194, 0223).
|
||||
//
|
||||
// Started with no arguments it is served by the node's runtime as the operator account, over MCP on stdio
|
||||
// (ADR 0188, ADR 0193): the module's tools and its health, and beside them the events the resolver's
|
||||
// answers give — a machine's name it now answers or no longer does, and a restart the watch made. Started
|
||||
// as `dnsmasq-tools watch` it is the module's long-running process, as root: it asks the resolver a real
|
||||
// question on every listen address and restarts it, within a budget, when it stops answering (watch.go,
|
||||
// novox/hq issue 348). stdout is the MCP channel; everything else is said on stderr.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 {
|
||||
if os.Args[1] != "watch" || len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: dnsmasq-tools [watch]")
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
m := Here()
|
||||
if err := m.Watch(ctx, ExecRunner, m.WriteState); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
m := Here()
|
||||
go Announce(m, stdio.Emit)
|
||||
if err := stdio.Serve("", Tools(m)); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// AnnounceEvery is how often the bundle reads the roster and the watch for what to say.
|
||||
var AnnounceEvery = 30 * time.Second
|
||||
|
||||
// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not
|
||||
// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when
|
||||
// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded.
|
||||
type Announcer struct {
|
||||
names map[string]string
|
||||
primed bool
|
||||
// said is every restart said, by when it was made; judged every one whose outcome was said.
|
||||
said, judged map[time.Time]bool
|
||||
}
|
||||
|
||||
// Event is one event to emit, and what saying it records.
|
||||
type Event struct {
|
||||
Type string
|
||||
Body map[string]any
|
||||
said func()
|
||||
}
|
||||
|
||||
// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does
|
||||
// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events;
|
||||
// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it
|
||||
// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names,
|
||||
// rather than every name removed. An event is remembered by Said, once it was emitted.
|
||||
func (a *Announcer) Next(names []Named, read bool, s State) []Event {
|
||||
if a.said == nil {
|
||||
a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{}
|
||||
}
|
||||
var out []Event
|
||||
if read {
|
||||
now := map[string]string{}
|
||||
for _, n := range names {
|
||||
now[n.Name] = n.Address
|
||||
}
|
||||
if !a.primed {
|
||||
a.names, a.primed = now, true
|
||||
} else {
|
||||
for name, address := range now {
|
||||
if _, had := a.names[name]; !had {
|
||||
out = append(out, Event{"name.added", map[string]any{"name": name, "address": address},
|
||||
func() { a.names[name] = address }})
|
||||
}
|
||||
}
|
||||
for name := range a.names {
|
||||
if _, has := now[name]; !has {
|
||||
out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range s.Restarts {
|
||||
at := r.At
|
||||
if !a.said[at] {
|
||||
body := map[string]any{"unit": Unit, "at": at, "why": r.Why}
|
||||
if r.Error != "" {
|
||||
body["error"] = r.Error
|
||||
}
|
||||
out = append(out, Event{"restarted", body, func() { a.said[at] = true }})
|
||||
}
|
||||
if r.Helped != nil && !a.judged[at] {
|
||||
out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped},
|
||||
func() { a.judged[at] = true }})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Said remembers an event as said.
|
||||
func (e Event) Said() {
|
||||
if e.said != nil {
|
||||
e.said()
|
||||
}
|
||||
}
|
||||
|
||||
// Announce says the events for as long as the bundle runs.
|
||||
func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) {
|
||||
time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs
|
||||
var a Announcer
|
||||
for {
|
||||
Tell(m, &a, emit)
|
||||
time.Sleep(AnnounceEvery)
|
||||
}
|
||||
}
|
||||
|
||||
// Tell is one read and what it says: an event that could not be emitted is said again at the next read.
|
||||
func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) {
|
||||
var names []Named
|
||||
text, err := m.read(NodesFile)
|
||||
if err == nil {
|
||||
names = AnsweredNames(text)
|
||||
}
|
||||
s, _, _ := m.ReadState()
|
||||
for _, e := range a.Next(names, err == nil, s) {
|
||||
if err := emit(e.Type, e.Body); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err)
|
||||
continue
|
||||
}
|
||||
e.Said()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Stale is how old the watch's last look may be before the health tool says it stopped looking.
|
||||
const Stale = 2 * time.Minute
|
||||
|
||||
// Health is the module's health answer (novox/hq ADR 0240): healthy only when the resolver answers its
|
||||
// real question on every listen address, over UDP and TCP, and the watch is looking.
|
||||
type Health struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why,omitempty"`
|
||||
Look Look `json:"look"`
|
||||
// Watch is what the watch last saw and did; absent when it has written nothing yet.
|
||||
Watch *State `json:"watch,omitempty"`
|
||||
}
|
||||
|
||||
// Health asks the resolver now, and reads what the watch did.
|
||||
func (m *Machine) Health(ctx context.Context) Health {
|
||||
l := m.Look(ctx)
|
||||
h := Health{Healthy: l.Healthy, Look: l}
|
||||
var why []string
|
||||
if !l.Healthy {
|
||||
why = append(why, "the resolver does not answer as it must: "+l.Why())
|
||||
}
|
||||
s, had, err := m.ReadState()
|
||||
switch {
|
||||
case err != nil:
|
||||
h.Healthy = false
|
||||
why = append(why, "what the watch saw cannot be read: "+err.Error())
|
||||
case !had:
|
||||
h.Healthy = false
|
||||
why = append(why, "the watch has written nothing yet, so nothing restarts the resolver when it stops answering")
|
||||
default:
|
||||
h.Watch = &s
|
||||
if age := m.Now().Sub(s.Last.At); age > Stale {
|
||||
h.Healthy = false
|
||||
why = append(why, fmt.Sprintf("the watch last looked %s ago, so nothing restarts the resolver when it "+
|
||||
"stops answering", age.Round(time.Second)))
|
||||
}
|
||||
if !l.Healthy {
|
||||
why = append(why, restartWords(s))
|
||||
}
|
||||
if s.Unit != "" && s.Unit != "active" {
|
||||
why = append(why, "the unit is "+s.Unit)
|
||||
}
|
||||
}
|
||||
h.Why = strings.Join(nonEmpty(why), "; ")
|
||||
return h
|
||||
}
|
||||
|
||||
// restartWords is what the watch did about it, in words.
|
||||
func restartWords(s State) string {
|
||||
if len(s.Restarts) == 0 {
|
||||
return "the watch has not restarted it" + orEmpty("; ", s.Held)
|
||||
}
|
||||
r := s.Restarts[len(s.Restarts)-1]
|
||||
words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC"))
|
||||
switch {
|
||||
case r.Error != "":
|
||||
words += ", and the restart failed: " + r.Error
|
||||
case r.Helped != nil && !*r.Helped:
|
||||
words += ", which did not help"
|
||||
case r.Helped == nil:
|
||||
words += ", and its next looks decide whether that helped"
|
||||
}
|
||||
if s.Held != "" {
|
||||
words += "; " + s.Held
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
func nonEmpty(s []string) []string {
|
||||
var out []string
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func str(description string) map[string]any {
|
||||
return map[string]any{"type": "string", "description": description}
|
||||
}
|
||||
|
||||
func arg(a map[string]any, k string) string {
|
||||
v, _ := a[k].(string)
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
// Tools are the module's tools.
|
||||
func Tools(m *Machine) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "dnsmasq_health", Description: "Whether this machine's mesh resolver answers: its real question — this " +
|
||||
"machine's own name — asked now on every address its configuration listens on, over UDP and TCP, each " +
|
||||
"answer with its code, addresses and time; and what the module's watch last saw, the restarts it made " +
|
||||
"within the hour, whether each helped, and why it holds back from another. Healthy only when every " +
|
||||
"question was answered rightly and the watch is looking (novox/hq ADR 0240, issue 348). (r)",
|
||||
Input: map[string]any{},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return m.Health(context.Background()), nil
|
||||
}},
|
||||
{Name: "dnsmasq_names", Description: "The mesh's machine names this machine's resolver answers, each with its " +
|
||||
"private address, as the roster wrote them (" + NodesFile + "). (r)",
|
||||
Input: map[string]any{},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
text, err := m.read(NodesFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the roster %s cannot be read: %w", NodesFile, err)
|
||||
}
|
||||
return map[string]any{"names": AnsweredNames(text)}, nil
|
||||
}},
|
||||
{Name: "dnsmasq_resolve", Description: "Ask this machine's resolver one name's IPv4 address, directly, on one " +
|
||||
"address it listens on (default every one) over udp or tcp (default both): the answer code, the addresses " +
|
||||
"and the time each took, or why none came — refused when nothing listens there. Replaces dig @address. (r)",
|
||||
Input: map[string]any{"name": str("the name to ask, e.g. plex.anchor.internal"),
|
||||
"address": str("one listen address to ask (default: every one in its configuration)"),
|
||||
"over": str("udp or tcp (default: both)")},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
return m.Resolve(context.Background(), arg(a, "name"), arg(a, "address"), arg(a, "over"))
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve asks name on one listen address or every one, over one transport or both.
|
||||
func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, error) {
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
config, err := m.read(ConfigFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err)
|
||||
}
|
||||
addresses := ListenAddresses(config)
|
||||
if address != "" {
|
||||
if !slices.Contains(addresses, address) {
|
||||
return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver",
|
||||
address, strings.Join(addresses, ", "))
|
||||
}
|
||||
addresses = []string{address}
|
||||
}
|
||||
overs := []string{"udp", "tcp"}
|
||||
switch over {
|
||||
case "":
|
||||
case "udp", "tcp":
|
||||
overs = []string{over}
|
||||
default:
|
||||
return nil, fmt.Errorf("over is udp or tcp, not %q", over)
|
||||
}
|
||||
var answers []Answer
|
||||
for _, a := range addresses {
|
||||
for _, o := range overs {
|
||||
answers = append(answers, m.Ask(ctx, a+":"+m.Port, o, name, AskWithin))
|
||||
}
|
||||
}
|
||||
return map[string]any{"name": name, "answers": answers}, nil
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The watch is the module's long-running process, as root. Every LookEvery it asks the resolver its real
|
||||
// question on every listen address (look.go); after Confirm failing looks in a row it restarts the
|
||||
// resolver's unit, within a budget, and writes what it saw and did for the tools, which say it to the mesh
|
||||
// (novox/hq issue 348).
|
||||
//
|
||||
// **Why the module restarts its own resolver.** A resolver that stops answering on a machine can cut that
|
||||
// machine off from the bus by name, and then nothing outside the machine — neither the gate's rollback nor
|
||||
// a healer in the controller — reaches it (ADR 0236 on the network path). The repair has to be made where
|
||||
// the resolver runs, as the identity provider repairs its own administrator (ADR 0224 §5). It is braked,
|
||||
// and it never says it worked: its next look does. A restart that did not help is said by the health
|
||||
// tool as the resolver's failure, which the node-engine states and the controller raises — the operator
|
||||
// hears of it only then.
|
||||
|
||||
// The watch's clocks and its budget.
|
||||
var (
|
||||
LookEvery = 15 * time.Second
|
||||
// Confirm failing looks in a row are a resolver that does not answer: one look can be wrong (issue 277).
|
||||
Confirm = 2
|
||||
// RestartGap is the least time between two restarts; RestartsPerHour the most within an hour.
|
||||
RestartGap = 2 * time.Minute
|
||||
RestartsPerHour = 3
|
||||
// HelpedWithin is how soon after a restart a healthy look says the restart helped.
|
||||
HelpedWithin = time.Minute
|
||||
)
|
||||
|
||||
// Unit is the resolver's service.
|
||||
const Unit = "dnsmasq.service"
|
||||
|
||||
// StateFile is where the watch writes what it saw, for the tools to read: readable by the operator
|
||||
// account the tools run as, written only by the watch.
|
||||
const StateFile = "/var/lib/dnsmasq-watch/watch.json"
|
||||
|
||||
// Restart is one restart the watch made.
|
||||
type Restart struct {
|
||||
At time.Time `json:"at"`
|
||||
Why string `json:"why"`
|
||||
// Error is what the service manager said when the restart itself failed.
|
||||
Error string `json:"error,omitempty"`
|
||||
// Helped is the next looks' word: true once a look within HelpedWithin answered rightly, false once
|
||||
// HelpedWithin passed without one, absent until then.
|
||||
Helped *bool `json:"helped,omitempty"`
|
||||
}
|
||||
|
||||
// State is the watch's last look and what it did, as the tools read it.
|
||||
type State struct {
|
||||
Last Look `json:"last"`
|
||||
// Failing is how many looks in a row failed; Since when the first of them looked.
|
||||
Failing int `json:"failing"`
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
// Restarts are the restarts within the last hour and the last one ever, newest last.
|
||||
Restarts []Restart `json:"restarts,omitempty"`
|
||||
// Held is why a restart that was due was not made: the budget is spent, or the unit is not running.
|
||||
Held string `json:"held,omitempty"`
|
||||
// Unit is the resolver unit's state at the last look, as the service manager said it: active,
|
||||
// activating, inactive, failed …
|
||||
Unit string `json:"unit,omitempty"`
|
||||
}
|
||||
|
||||
// Runner runs one command; injected, so the watch is tested over recorded answers.
|
||||
type Runner func(ctx context.Context, name string, args ...string) (string, error)
|
||||
|
||||
// ExecRunner runs a command on the machine, bounded to a minute.
|
||||
func ExecRunner(ctx context.Context, name string, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
var out bytes.Buffer
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Stdout, cmd.Stderr = &out, &out
|
||||
err := cmd.Run()
|
||||
return strings.TrimSpace(out.String()), err
|
||||
}
|
||||
|
||||
// Step is one look of the watch and what follows from it: the state after, and whether to restart and why.
|
||||
// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of
|
||||
// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at
|
||||
// boot for the private address — or failed is the service manager's, and none of them is a resolver that
|
||||
// runs and does not answer. Pure.
|
||||
func Step(s State, l Look, unit string) (State, string) {
|
||||
now := l.At
|
||||
s.Last, s.Unit = l, unit
|
||||
// What the earlier restarts are now known to have done.
|
||||
for i := range s.Restarts {
|
||||
r := &s.Restarts[i]
|
||||
if r.Helped != nil || r.Error != "" {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case l.Healthy && now.Sub(r.At) <= HelpedWithin:
|
||||
yes := true
|
||||
r.Helped = &yes
|
||||
case now.Sub(r.At) > HelpedWithin:
|
||||
no := false
|
||||
r.Helped = &no
|
||||
}
|
||||
}
|
||||
// The budget's window: the last hour, and always the newest.
|
||||
kept := s.Restarts[:0]
|
||||
for i, r := range s.Restarts {
|
||||
if now.Sub(r.At) <= time.Hour || i == len(s.Restarts)-1 {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
s.Restarts = kept
|
||||
if l.Healthy {
|
||||
s.Failing, s.Since, s.Held = 0, nil, ""
|
||||
return s, ""
|
||||
}
|
||||
if s.Failing == 0 {
|
||||
at := now
|
||||
s.Since = &at
|
||||
}
|
||||
s.Failing++
|
||||
if s.Failing < Confirm || len(l.Problems) > 0 && len(l.Answers) == 0 {
|
||||
// Not yet confirmed; or the look could not ask at all — a restart answers nothing a missing
|
||||
// configuration or roster is missing.
|
||||
return s, ""
|
||||
}
|
||||
if unit != "active" {
|
||||
s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+
|
||||
"answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit))
|
||||
return s, ""
|
||||
}
|
||||
within := 0
|
||||
var newest *Restart
|
||||
for i, r := range s.Restarts {
|
||||
if now.Sub(r.At) <= time.Hour {
|
||||
within++
|
||||
}
|
||||
newest = &s.Restarts[i]
|
||||
}
|
||||
switch {
|
||||
case newest != nil && now.Sub(newest.At) < RestartGap:
|
||||
s.Held = ""
|
||||
return s, "" // the last restart is still settling
|
||||
case within >= RestartsPerHour:
|
||||
s.Held = fmt.Sprintf("%d restart(s) within the last hour, the most the watch makes; it restarts nothing "+
|
||||
"more until the oldest is an hour old, and the resolver's failure is said as its health", within)
|
||||
return s, ""
|
||||
}
|
||||
s.Held = ""
|
||||
return s, fmt.Sprintf("%d look(s) in a row since %s failed: %s", s.Failing, s.Since.Format("15:04:05 UTC"), l.Why())
|
||||
}
|
||||
|
||||
// Watch runs the watch until ctx ends.
|
||||
func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error) error {
|
||||
var s State
|
||||
if raw, err := m.read(StateFile); err == nil {
|
||||
_ = json.Unmarshal([]byte(raw), &s) // the budget survives the watch's own restart
|
||||
}
|
||||
tick := time.NewTicker(LookEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
l := m.Look(ctx)
|
||||
unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit)
|
||||
if err != nil {
|
||||
unit = ""
|
||||
}
|
||||
next, why := Step(s, l, strings.TrimSpace(unit))
|
||||
if why != "" {
|
||||
r := Restart{At: m.Now().UTC(), Why: why}
|
||||
// Not waited for: a start that waits for the private address is the service manager's to time,
|
||||
// and the next looks say whether it answers.
|
||||
if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil {
|
||||
r.Error = strings.TrimSpace(err.Error() + " " + out)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error))
|
||||
next.Restarts = append(next.Restarts, r)
|
||||
} else if !l.Healthy && next.Failing == Confirm {
|
||||
fmt.Fprintf(os.Stderr, "dnsmasq-watch: the resolver does not answer: %s%s\n", l.Why(), orEmpty("; ", next.Held))
|
||||
}
|
||||
s = next
|
||||
if err := write(s); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "dnsmasq-watch: what it saw could not be written:", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if s == "" {
|
||||
return "not known (the service manager did not say)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func orEmpty(prefix, s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return prefix + s
|
||||
}
|
||||
|
||||
// WriteState writes the state whole, replacing the file in one rename.
|
||||
func (m *Machine) WriteState(s State) error {
|
||||
path := filepath.Join(m.Root, StateFile)
|
||||
body, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, body, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// ReadState is the watch's last state, and false when it has written none.
|
||||
func (m *Machine) ReadState() (State, bool, error) {
|
||||
raw, err := m.read(StateFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return State{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return State{}, false, err
|
||||
}
|
||||
var s State
|
||||
if err := json.Unmarshal([]byte(raw), &s); err != nil {
|
||||
return State{}, false, fmt.Errorf("%s is not the watch's: %w", StateFile, err)
|
||||
}
|
||||
return s, true, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module dnsmasq
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||
@@ -0,0 +1,2 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
@@ -1,38 +0,0 @@
|
||||
// dnsmasq's events. The resolver's answered set changes whenever the mesh rewrites the wildcard file
|
||||
// and restarts it — a machine joined the private network or left it. This watches that file and
|
||||
// announces, from the resolver's own vantage, that a name became (or stopped being) resolvable on
|
||||
// this node: DNS having actually propagated here, distinct from the mesh's own node.* events.
|
||||
//
|
||||
// Emits (novox/hq ADR 0041/0042):
|
||||
// module.dnsmasq.name.added — this node's resolver now answers a machine's name
|
||||
// module.dnsmasq.name.removed — it no longer does
|
||||
|
||||
import { emit } from "@novox/mesh-sdk/events";
|
||||
import { DnsmasqClient } from "./client.js";
|
||||
|
||||
const dnsmasq = DnsmasqClient.fromEnv();
|
||||
|
||||
// name -> address, primed silently so a restart does not re-announce every name it already answered.
|
||||
const known = new Map<string, string>();
|
||||
let primed = false;
|
||||
|
||||
async function poll(): Promise<void> {
|
||||
const now = new Map((await dnsmasq.answeredNames()).map((a) => [a.name, a.address]));
|
||||
if (primed) {
|
||||
for (const [name, address] of now) {
|
||||
if (!known.has(name)) await emit("name.added", { name, address });
|
||||
}
|
||||
for (const [name] of known) {
|
||||
if (!now.has(name)) await emit("name.removed", { name });
|
||||
}
|
||||
}
|
||||
known.clear();
|
||||
for (const [name, address] of now) known.set(name, address);
|
||||
primed = true;
|
||||
}
|
||||
|
||||
const run = (): void => void poll().catch((err) => console.error(`[dnsmasq] ${err}`));
|
||||
setInterval(run, 30_000);
|
||||
run();
|
||||
|
||||
console.log("[dnsmasq] watching the resolver's answered names");
|
||||
File diff suppressed because one or more lines are too long
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"name": "@novox/module-dnsmasq",
|
||||
"version": "0.1.0",
|
||||
"description": "dnsmasq — the mesh's resolver: answers wildcard node names and forwards the rest upstream. Its tools and events live here.",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"dependencies": { "@novox/mesh-sdk": "^0.1.0" },
|
||||
"devDependencies": { "@types/node": "^22.0.0", "typescript": "^5.6.0" }
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
// dnsmasq's tools — a resolver's two useful questions: what does it answer, and does it answer a
|
||||
// given name. Moved into the module (novox/hq ADR 0039); the mesh serves them through the sdk.
|
||||
|
||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import { DnsmasqClient } from "../client.js";
|
||||
|
||||
export function getDnsmasqTools(dnsmasq: DnsmasqClient): ToolDefinition[] {
|
||||
return [
|
||||
{
|
||||
name: "dnsmasq_names",
|
||||
description: "The mesh names this node's resolver answers — each a machine and everything under it.",
|
||||
input: {},
|
||||
run: async () => ({ names: await dnsmasq.answeredNames() }),
|
||||
},
|
||||
{
|
||||
name: "dnsmasq_resolve",
|
||||
description: "Resolve a mesh name through this node's own resolver — the check that wildcard-resolution answers.",
|
||||
input: { name: { type: "string", description: "a name to resolve, e.g. plex.anchor.internal" } },
|
||||
run: async (args) => {
|
||||
const name = String(args.name);
|
||||
try {
|
||||
return { name, addresses: await dnsmasq.resolve(name) };
|
||||
} catch (err) {
|
||||
return { name, addresses: [], error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
registerModuleTools("dnsmasq", (env) => getDnsmasqTools(DnsmasqClient.fromEnv(env)));
|
||||
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["client.ts", "index.ts", "tools/index.ts"]
|
||||
}
|
||||
@@ -991,30 +991,53 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
|
||||
}
|
||||
|
||||
// Events is what the runtime did in a window ending now, the latest last.
|
||||
//
|
||||
// **The runtime keeps only its last events in memory** (about 256), and the images' own health checks
|
||||
// make three exec events each per look: on a machine with a few checked containers that history is ten
|
||||
// minutes long or less, all of it exec events (hq research 032). A window asked for further back than
|
||||
// the history reaches is answered with what is still held AND says where the history begins, so an
|
||||
// empty list never reads as "nothing happened" (hq issue 348: three hours asked across an outage,
|
||||
// answered zero). The type filter is applied here rather than by the runtime, for the same reason: the
|
||||
// history's start is only known from all of it.
|
||||
func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
|
||||
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
|
||||
return c.eventsAt(ctx, time.Now(), minutes, kind, limit, execs)
|
||||
}
|
||||
|
||||
func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind string, limit int, execs bool) (map[string]any, error) {
|
||||
if kind != "" {
|
||||
switch kind {
|
||||
case "container", "image", "network", "volume", "daemon", "plugin", "builder":
|
||||
default:
|
||||
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
|
||||
}
|
||||
args = append(args, "--filter", "type="+kind)
|
||||
}
|
||||
// The whole of what the runtime holds, from the epoch: its buffer, which says where its history really
|
||||
// begins and whether it is full; the window is cut from it here.
|
||||
args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"}
|
||||
out, err := c.docker(ctx, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := jsonLines[runtimeEvent](out)
|
||||
all, err := jsonLines[runtimeEvent](out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start := now.Add(-time.Duration(minutes) * time.Minute)
|
||||
raw := make([]runtimeEvent, 0, len(all))
|
||||
for _, e := range all {
|
||||
if !time.Unix(0, e.TimeNano).Before(start) {
|
||||
raw = append(raw, e)
|
||||
}
|
||||
}
|
||||
// An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer
|
||||
// is read by agents and kept in their transcripts, which would make it a second copy of the leak.
|
||||
envs := &envCache{c: c, ctx: ctx}
|
||||
redacted := map[string]bool{}
|
||||
events := []map[string]any{}
|
||||
for _, e := range raw {
|
||||
if kind != "" && e.Type != kind {
|
||||
continue
|
||||
}
|
||||
if !execs && strings.HasPrefix(e.Action, "exec_") {
|
||||
continue
|
||||
}
|
||||
@@ -1042,14 +1065,50 @@ func (c *Client) Events(ctx context.Context, minutes int, kind string, limit int
|
||||
if len(events) > limit {
|
||||
events = events[len(events)-limit:]
|
||||
}
|
||||
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events}
|
||||
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
|
||||
"held": len(all)}
|
||||
if len(redacted) > 0 {
|
||||
answer["redacted"] = sortedSet(redacted)
|
||||
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
|
||||
}
|
||||
for k, v := range historyReach(all, start) {
|
||||
answer[k] = v
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer
|
||||
// holding fewer has every event since the runtime started; a full one has lost what came before its oldest.
|
||||
const RuntimeEventsKept = 256
|
||||
|
||||
// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window
|
||||
// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not
|
||||
// full — then it holds everything since the runtime started. Otherwise where the history begins, and that
|
||||
// nothing before it can be answered from here.
|
||||
func historyReach(all []runtimeEvent, start time.Time) map[string]any {
|
||||
if len(all) == 0 {
|
||||
return map[string]any{"complete": true, "oldest_held": nil,
|
||||
"history": "the runtime holds no events at all: none happened since it started"}
|
||||
}
|
||||
oldest := time.Unix(0, all[0].TimeNano)
|
||||
for _, e := range all {
|
||||
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
|
||||
oldest = t
|
||||
}
|
||||
}
|
||||
at := oldest.UTC().Format(time.RFC3339)
|
||||
if len(all) < RuntimeEventsKept {
|
||||
return map[string]any{"complete": true, "oldest_held": at,
|
||||
"history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)}
|
||||
}
|
||||
if !oldest.After(start) {
|
||||
return map[string]any{"complete": true, "oldest_held": at}
|
||||
}
|
||||
return map[string]any{"complete": false, "oldest_held": at,
|
||||
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
|
||||
len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
|
||||
}
|
||||
|
||||
func sortedSet(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"reflect"
|
||||
@@ -299,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
|
||||
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
|
||||
`
|
||||
f := (&fake{}).on("docker events", Ran{Stdout: out})
|
||||
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
|
||||
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -307,7 +308,8 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
|
||||
if len(evs) != 1 || evs[0]["action"] != "die" || evs[0]["mesh_held"] != true || evs[0]["exit_code"] != "137" {
|
||||
t.Fatalf("%v", evs)
|
||||
}
|
||||
if !f.ran("docker events --since 30m --until 0s --format {{json .}} --filter type=container") {
|
||||
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
|
||||
if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
|
||||
t.Fatalf("%+v", f.calls)
|
||||
}
|
||||
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
|
||||
@@ -362,3 +364,70 @@ func TestProblemsNameWhyAndUnlabelledIsTheCleanupList(t *testing.T) {
|
||||
t.Fatalf("%v %v", u, err)
|
||||
}
|
||||
}
|
||||
|
||||
// eventLine is one line of `docker events --format '{{json .}}'`.
|
||||
func eventLine(at time.Time, kind, action, name string) string {
|
||||
return fmt.Sprintf(`{"Type":%q,"Action":%q,"Actor":{"ID":"abcdef0123456789","Attributes":{"name":%q}},"timeNano":%d}`+"\n",
|
||||
kind, action, name, at.UnixNano())
|
||||
}
|
||||
|
||||
// Issue 348: three hours asked across an outage answered zero, because the runtime's history held only the
|
||||
// last ten minutes of health-check execs. The answer must say the history is short, not that nothing happened.
|
||||
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
|
||||
var out strings.Builder
|
||||
for i := 0; i < RuntimeEventsKept; i++ {
|
||||
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
|
||||
}
|
||||
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["count"] != 0 || got["held"] != RuntimeEventsKept {
|
||||
t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept)
|
||||
}
|
||||
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
|
||||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
|
||||
t.Fatalf("an empty answer from a short history reads as quiet: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
|
||||
// A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer
|
||||
// is not full, so it holds every event since the runtime started.
|
||||
out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") +
|
||||
eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
|
||||
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
|
||||
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
|
||||
f := (&fake{}).on("docker events", Ran{Stdout: out})
|
||||
got, err := client(f, 1000).eventsAt(context.Background(), now, 60, "network", 200, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") {
|
||||
t.Fatalf("a quiet machine's window: %v", got)
|
||||
}
|
||||
if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
|
||||
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
|
||||
}
|
||||
// A quiet machine asked for ten minutes, its newest event an hour old: complete, and none.
|
||||
got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false)
|
||||
if got["complete"] != true || got["count"] != 0 {
|
||||
t.Fatalf("a quiet machine read as one whose history was lost: %v", got)
|
||||
}
|
||||
// No events at all: none since the runtime started.
|
||||
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
|
||||
if got["complete"] != true || got["count"] != 0 {
|
||||
t.Fatalf("an empty buffer: %v", got)
|
||||
}
|
||||
// A full buffer whose oldest is before the window: complete.
|
||||
var full strings.Builder
|
||||
for i := 0; i < RuntimeEventsKept; i++ {
|
||||
full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x"))
|
||||
}
|
||||
got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
|
||||
if got["complete"] != true || got["history"] != nil {
|
||||
t.Fatalf("a full buffer reaching past the window: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +70,19 @@ func tools(c *Client) []stdio.Tool {
|
||||
return c.Inspect(ctx, ref)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "docker_resolv_conf",
|
||||
Description: "The resolver file one container really uses (/etc/resolv.conf as it sees it, read through the runtime's archive, no exec): " +
|
||||
"its nameservers, search and options and the raw text, beside the machine's own nameservers and whether they are the same.",
|
||||
Input: map[string]any{"container": containerArg},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ref, err := text(args, "container")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.ResolvConfOf(ctx, ref)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "docker_logs",
|
||||
Description: "The last lines one container wrote, both streams merged in order, each with its timestamp (default 200, at most 2000 lines; a line is cut at 4 KiB). " +
|
||||
@@ -262,7 +275,8 @@ func tools(c *Client) []stdio.Tool {
|
||||
{
|
||||
Name: "docker_events",
|
||||
Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " +
|
||||
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag.",
|
||||
"an exec's command line is shown with any secret it carried as [redacted: <what it was>] — a value of the container's environment named like a secret, a password in a URI, the word after a password flag. " +
|
||||
"The runtime keeps only its last ~256 events in memory, which health-check execs fill in minutes: complete=false and history say where what it holds begins, and an empty list then means gone, not quiet.",
|
||||
Input: map[string]any{
|
||||
"minutes": map[string]any{"type": "integer", "description": "how far back (default 60, at most 1440)"},
|
||||
"type": map[string]any{"type": "string", "description": "only one kind: container, image, network, volume, daemon, plugin or builder"},
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
// What resolver a container really asks (hq issue 348). A container gets its own copy of the machine's
|
||||
// resolver file, made by the runtime when it is created and kept under the runtime's root-only
|
||||
// directory; a container on the host's network gets a copy too. During the 2026-10-09 outage the
|
||||
// control node's route-proxy failed every lookup against 10.10.0.1 alone for twenty minutes while the
|
||||
// machine's file listed both mesh resolvers — and nothing could show what its copy said. This reads the
|
||||
// file as the container sees it, through the runtime's archive endpoint (`docker cp … -`, a tar stream):
|
||||
// no exec into the container, nothing written, and it works where the program has no shell.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ContainerResolvConf is the path read inside a container.
|
||||
const ContainerResolvConf = "/etc/resolv.conf"
|
||||
|
||||
// HostResolvConf is the machine's own resolver file, read for comparison.
|
||||
const HostResolvConf = "/etc/resolv.conf"
|
||||
|
||||
// ResolvConf is a resolver file, parsed.
|
||||
type ResolvConf struct {
|
||||
Nameservers []string `json:"nameservers"`
|
||||
Search []string `json:"search,omitempty"`
|
||||
Options []string `json:"options,omitempty"`
|
||||
Raw string `json:"raw"`
|
||||
}
|
||||
|
||||
// parseResolvConf reads nameserver, search (and domain) and options lines; comments are kept in Raw only.
|
||||
func parseResolvConf(text string) ResolvConf {
|
||||
out := ResolvConf{Nameservers: []string{}, Raw: text}
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 0 || strings.HasPrefix(f[0], "#") || strings.HasPrefix(f[0], ";") {
|
||||
continue
|
||||
}
|
||||
switch f[0] {
|
||||
case "nameserver":
|
||||
if len(f) > 1 {
|
||||
out.Nameservers = append(out.Nameservers, f[1])
|
||||
}
|
||||
case "search", "domain":
|
||||
out.Search = append(out.Search, f[1:]...)
|
||||
case "options":
|
||||
out.Options = append(out.Options, f[1:]...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fileFromArchive is the one regular file a `docker cp <ref>:<path> -` stream carries.
|
||||
func fileFromArchive(stream string) (string, error) {
|
||||
tr := tar.NewReader(strings.NewReader(stream))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return "", errors.New("the runtime's archive held no file")
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
|
||||
}
|
||||
switch h.Typeflag {
|
||||
case tar.TypeReg:
|
||||
body, err := io.ReadAll(io.LimitReader(tr, 64<<10))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the runtime's archive could not be read: %w", err)
|
||||
}
|
||||
return string(body), nil
|
||||
case tar.TypeSymlink:
|
||||
return "", fmt.Errorf("inside the container %s is a link to %s, which the runtime's archive does not follow", h.Name, h.Linkname)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ResolvConfOf is a container's resolver file as it sees it, beside the machine's own, and whether they list
|
||||
// the same resolvers in the same order.
|
||||
func (c *Client) ResolvConfOf(ctx context.Context, ref string) (map[string]any, error) {
|
||||
ref, err := Ref(ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := map[string]any{"container": ref, "path": ContainerResolvConf}
|
||||
stream, err := c.docker(ctx, "cp", ref+":"+ContainerResolvConf, "-")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
|
||||
}
|
||||
text, err := fileFromArchive(stream)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the container's %s could not be read: %w", ContainerResolvConf, err)
|
||||
}
|
||||
inside := parseResolvConf(text)
|
||||
answer["resolv_conf"] = inside
|
||||
if raw, err := c.ReadFile(HostResolvConf); err != nil {
|
||||
answer["machine"] = "the machine's " + HostResolvConf + " could not be read for comparison: " + err.Error()
|
||||
} else {
|
||||
host := parseResolvConf(string(raw))
|
||||
answer["machine_nameservers"] = host.Nameservers
|
||||
same := strings.Join(host.Nameservers, " ") == strings.Join(inside.Nameservers, " ")
|
||||
answer["same_as_machine"] = same
|
||||
if !same {
|
||||
answer["note"] = "the container asks other resolvers than the machine lists: its copy was made when it was created, " +
|
||||
"or the runtime's dns setting chose them; a container is given the machine's current file only when it is created again"
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func archiveOf(t *testing.T, h tar.Header, body string) string {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := tar.NewWriter(&buf)
|
||||
h.Size = int64(len(body))
|
||||
if h.Typeflag == tar.TypeSymlink {
|
||||
h.Size = 0
|
||||
}
|
||||
if err := w.WriteHeader(&h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.Typeflag != tar.TypeSymlink {
|
||||
if _, err := w.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
w.Close()
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// Issue 348: route-proxy asked 10.10.0.1 alone while the machine listed both mesh resolvers; its copy
|
||||
// of the file could not be read by anything on the mesh.
|
||||
func TestAContainersResolverFileIsReadThroughTheArchiveAndComparedWithTheMachines(t *testing.T) {
|
||||
inside := "# Generated by Docker Engine.\nnameserver 10.10.0.1\nsearch novox.internal\noptions timeout:1 attempts:2\n"
|
||||
f := (&fake{}).on("docker cp route-proxy:/etc/resolv.conf -",
|
||||
Ran{Stdout: archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, inside)})
|
||||
c := client(f, 1000)
|
||||
c.ReadFile = func(p string) ([]byte, error) {
|
||||
if p != HostResolvConf {
|
||||
t.Fatalf("read %s", p)
|
||||
}
|
||||
return []byte("nameserver 10.10.0.1\nnameserver 10.10.0.2\noptions timeout:1 attempts:2 edns0\n"), nil
|
||||
}
|
||||
got, err := c.ResolvConfOf(context.Background(), "route-proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rc := got["resolv_conf"].(ResolvConf)
|
||||
if strings.Join(rc.Nameservers, ",") != "10.10.0.1" || strings.Join(rc.Search, ",") != "novox.internal" ||
|
||||
strings.Join(rc.Options, ",") != "timeout:1,attempts:2" || rc.Raw != inside {
|
||||
t.Fatalf("parsed: %+v", rc)
|
||||
}
|
||||
if got["same_as_machine"] != false || got["note"] == nil {
|
||||
t.Fatalf("a copy listing fewer resolvers than the machine is not said: %v", got)
|
||||
}
|
||||
if f.ran("docker exec") {
|
||||
t.Fatal("read by exec, not through the archive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
|
||||
c := client((&fake{}).on("docker cp", Ran{Status: 1, Stderr: "Error response from daemon: No such container: nope\n"}), 1000)
|
||||
if _, err := c.ResolvConfOf(context.Background(), "nope"); err == nil || !strings.Contains(err.Error(), "could not be read") {
|
||||
t.Fatalf("a missing container: %v", err)
|
||||
}
|
||||
link := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeSymlink, Linkname: "/run/systemd/resolve/stub-resolv.conf"}, "")
|
||||
c = client((&fake{}).on("docker cp", Ran{Stdout: link}), 1000)
|
||||
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil || !strings.Contains(err.Error(), "link to") {
|
||||
t.Fatalf("a link inside: %v", err)
|
||||
}
|
||||
c = client((&fake{}).on("docker cp", Ran{Stdout: "not a tar"}), 1000)
|
||||
if _, err := c.ResolvConfOf(context.Background(), "x"); err == nil {
|
||||
t.Fatal("garbage was read as a file")
|
||||
}
|
||||
ok := archiveOf(t, tar.Header{Name: "resolv.conf", Typeflag: tar.TypeReg, Mode: 0o644}, "nameserver 10.10.0.1\n")
|
||||
c = client((&fake{}).on("docker cp", Ran{Stdout: ok}), 1000)
|
||||
c.ReadFile = func(string) ([]byte, error) { return nil, errors.New("permission denied") }
|
||||
got, err := c.ResolvConfOf(context.Background(), "x")
|
||||
if err != nil || !strings.Contains(got["machine"].(string), "could not be read") || got["same_as_machine"] != nil {
|
||||
t.Fatalf("an unreadable machine file: %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The container is named as docker knows it, never as something docker would read as an option.
|
||||
func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) {
|
||||
f := &fake{}
|
||||
for _, ref := range []string{"--privileged", "a b", "", "x;rm"} {
|
||||
if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil {
|
||||
t.Errorf("%q was asked of the runtime", ref)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Fatalf("the runtime was asked: %+v", f.calls)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The shape of the leak in hq issue 268: a server password announced at start and a database URI
|
||||
@@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) {
|
||||
f := (&fake{}).
|
||||
on("docker events", Ran{Stdout: execEvents}).
|
||||
on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"})
|
||||
got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true)
|
||||
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"tools": [
|
||||
"docker_list",
|
||||
"docker_inspect",
|
||||
"docker_resolv_conf",
|
||||
"docker_logs",
|
||||
"docker_secrets_in_logs",
|
||||
"docker_secrets_in_events",
|
||||
@@ -38,6 +39,11 @@
|
||||
"docker_problems",
|
||||
"docker_ports"
|
||||
],
|
||||
"replaces": {
|
||||
"docker_resolv_conf": [
|
||||
"docker exec cat /etc/resolv.conf"
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package main
|
||||
|
||||
// Asking one named resolver directly (hq issue 348). netcheck_dns without a server asks the machine's
|
||||
// resolver the way its programs do — resolv.conf, hosts, search domains, several servers in turn — which
|
||||
// answers "can this machine resolve?" and hides which resolver answered. During the 2026-10-09 outage the
|
||||
// question was "does the resolver at 10.10.0.1 answer?", and nothing on the mesh could ask it. Asked with a
|
||||
// server, the question goes to that address and port alone, over UDP or TCP, and the answer says its
|
||||
// response code: a refusal, no such name, and no answer at all are three different findings.
|
||||
//
|
||||
// The wire format is written here rather than taken from a library: a question and the records of five
|
||||
// types are all it needs, and the module stays without a dependency.
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var dnsTypeCodes = map[string]uint16{"A": 1, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28}
|
||||
|
||||
var rcodeWords = map[int]string{0: "NOERROR", 1: "FORMERR", 2: "SERVFAIL", 3: "NXDOMAIN", 4: "NOTIMP", 5: "REFUSED"}
|
||||
|
||||
// serverAddress is a server as asked — an address, with a port or without (53) — as host:port.
|
||||
func serverAddress(server string) (string, error) {
|
||||
server = strings.TrimSpace(server)
|
||||
if host, port, err := net.SplitHostPort(server); err == nil {
|
||||
p, err := strconv.Atoi(port)
|
||||
if err != nil || p < 1 || p > 65535 {
|
||||
return "", fmt.Errorf("server %q: the port is not one (1-65535)", server)
|
||||
}
|
||||
if net.ParseIP(host) == nil {
|
||||
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
|
||||
}
|
||||
return net.JoinHostPort(host, port), nil
|
||||
}
|
||||
if net.ParseIP(server) == nil {
|
||||
return "", fmt.Errorf("server %q: an address, not a name — a resolver is asked by the address it listens on", server)
|
||||
}
|
||||
return net.JoinHostPort(server, "53"), nil
|
||||
}
|
||||
|
||||
// askServer asks one resolver one question and reads its answer.
|
||||
func askServer(server, name, kind string, overTCP bool, timeout time.Duration) (rcode int, answers []string, err error) {
|
||||
addr, err := serverAddress(server)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
query, id, err := buildQuery(name, dnsTypeCodes[kind])
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
network := "udp"
|
||||
if overTCP {
|
||||
network = "tcp"
|
||||
}
|
||||
conn, err := net.DialTimeout(network, addr, timeout)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(time.Now().Add(timeout))
|
||||
var reply []byte
|
||||
if overTCP {
|
||||
framed := binary.BigEndian.AppendUint16(nil, uint16(len(query)))
|
||||
if _, err := conn.Write(append(framed, query...)); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
var size [2]byte
|
||||
if _, err := readFull(conn, size[:]); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
reply = make([]byte, binary.BigEndian.Uint16(size[:]))
|
||||
if _, err := readFull(conn, reply); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
} else {
|
||||
if _, err := conn.Write(query); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
buf := make([]byte, 4096)
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
reply = buf[:n]
|
||||
}
|
||||
return parseReply(reply, id, dnsTypeCodes[kind])
|
||||
}
|
||||
|
||||
func readFull(conn net.Conn, b []byte) (int, error) {
|
||||
got := 0
|
||||
for got < len(b) {
|
||||
n, err := conn.Read(b[got:])
|
||||
got += n
|
||||
if err != nil {
|
||||
return got, err
|
||||
}
|
||||
}
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// buildQuery is one question, recursion desired, under a random id.
|
||||
func buildQuery(name string, qtype uint16) ([]byte, uint16, error) {
|
||||
var idb [2]byte
|
||||
_, _ = rand.Read(idb[:])
|
||||
id := binary.BigEndian.Uint16(idb[:])
|
||||
msg := binary.BigEndian.AppendUint16(nil, id)
|
||||
msg = append(msg, 0x01, 0x00) // recursion desired
|
||||
msg = append(msg, 0, 1, 0, 0, 0, 0, 0, 0) // one question
|
||||
wire, err := encodeName(name)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
msg = append(msg, wire...)
|
||||
msg = binary.BigEndian.AppendUint16(msg, qtype)
|
||||
msg = binary.BigEndian.AppendUint16(msg, 1) // IN
|
||||
return msg, id, nil
|
||||
}
|
||||
|
||||
func encodeName(name string) ([]byte, error) {
|
||||
name = strings.TrimSuffix(strings.TrimSpace(name), ".")
|
||||
var out []byte
|
||||
if name != "" {
|
||||
for _, label := range strings.Split(name, ".") {
|
||||
if label == "" || len(label) > 63 {
|
||||
return nil, fmt.Errorf("name %q has an empty or too long label", name)
|
||||
}
|
||||
out = append(out, byte(len(label)))
|
||||
out = append(out, label...)
|
||||
}
|
||||
}
|
||||
return append(out, 0), nil
|
||||
}
|
||||
|
||||
var errShort = errors.New("the answer is cut short")
|
||||
|
||||
// readName reads a possibly compressed name at off, and the offset after it.
|
||||
func readName(msg []byte, off int) (string, int, error) {
|
||||
var labels []string
|
||||
next, jumped := -1, 0
|
||||
for {
|
||||
if off >= len(msg) {
|
||||
return "", 0, errShort
|
||||
}
|
||||
l := int(msg[off])
|
||||
switch {
|
||||
case l == 0:
|
||||
if next < 0 {
|
||||
next = off + 1
|
||||
}
|
||||
return strings.Join(labels, "."), next, nil
|
||||
case l&0xC0 == 0xC0:
|
||||
if off+1 >= len(msg) {
|
||||
return "", 0, errShort
|
||||
}
|
||||
if next < 0 {
|
||||
next = off + 2
|
||||
}
|
||||
if jumped++; jumped > 32 {
|
||||
return "", 0, errors.New("the answer's names point in a loop")
|
||||
}
|
||||
off = int(binary.BigEndian.Uint16(msg[off:]) & 0x3FFF)
|
||||
default:
|
||||
if off+1+l > len(msg) {
|
||||
return "", 0, errShort
|
||||
}
|
||||
labels = append(labels, string(msg[off+1:off+1+l]))
|
||||
off += 1 + l
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseReply reads the response code and the answers of the asked type.
|
||||
func parseReply(msg []byte, id, qtype uint16) (int, []string, error) {
|
||||
if len(msg) < 12 {
|
||||
return 0, nil, errShort
|
||||
}
|
||||
if binary.BigEndian.Uint16(msg) != id {
|
||||
return 0, nil, errors.New("the answer is to another question (its id differs)")
|
||||
}
|
||||
if msg[2]&0x80 == 0 {
|
||||
return 0, nil, errors.New("the answer is not marked as one")
|
||||
}
|
||||
rcode := int(msg[3] & 0x0F)
|
||||
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
|
||||
off := 12
|
||||
for i := 0; i < qd; i++ {
|
||||
_, n, err := readName(msg, off)
|
||||
if err != nil {
|
||||
return rcode, nil, err
|
||||
}
|
||||
off = n + 4
|
||||
}
|
||||
answers := []string{}
|
||||
for i := 0; i < an; i++ {
|
||||
_, n, err := readName(msg, off)
|
||||
if err != nil {
|
||||
return rcode, answers, err
|
||||
}
|
||||
off = n
|
||||
if off+10 > len(msg) {
|
||||
return rcode, answers, errShort
|
||||
}
|
||||
rtype := binary.BigEndian.Uint16(msg[off:])
|
||||
size := int(binary.BigEndian.Uint16(msg[off+8:]))
|
||||
data := off + 10
|
||||
if data+size > len(msg) {
|
||||
return rcode, answers, errShort
|
||||
}
|
||||
rdata := msg[data : data+size]
|
||||
off = data + size
|
||||
if rtype != qtype {
|
||||
continue // a CNAME on the way to an address, say
|
||||
}
|
||||
switch rtype {
|
||||
case 1, 28:
|
||||
answers = append(answers, net.IP(rdata).String())
|
||||
case 5:
|
||||
s, _, err := readName(msg, data)
|
||||
if err != nil {
|
||||
return rcode, answers, err
|
||||
}
|
||||
answers = append(answers, s)
|
||||
case 15:
|
||||
if size < 3 {
|
||||
return rcode, answers, errShort
|
||||
}
|
||||
s, _, err := readName(msg, data+2)
|
||||
if err != nil {
|
||||
return rcode, answers, err
|
||||
}
|
||||
answers = append(answers, fmt.Sprintf("%d %s", binary.BigEndian.Uint16(rdata), s))
|
||||
case 16:
|
||||
var parts []string
|
||||
for p := 0; p < len(rdata); {
|
||||
l := int(rdata[p])
|
||||
if p+1+l > len(rdata) {
|
||||
return rcode, answers, errShort
|
||||
}
|
||||
parts = append(parts, string(rdata[p+1:p+1+l]))
|
||||
p += 1 + l
|
||||
}
|
||||
answers = append(answers, strings.Join(parts, ""))
|
||||
}
|
||||
}
|
||||
return rcode, answers, nil
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// respond answers one query: novox.internal A 10.10.0.1 via a compressed name, anything else rcode.
|
||||
func respond(q []byte, rcode byte) []byte {
|
||||
end := 12
|
||||
for q[end] != 0 {
|
||||
end += int(q[end]) + 1
|
||||
}
|
||||
question := q[12 : end+5]
|
||||
name := strings.ToLower(string(q[13:end]))
|
||||
r := append([]byte{}, q[:2]...)
|
||||
r = append(r, 0x81, 0x80|rcode, 0, 1)
|
||||
answer := rcode == 0 && strings.Contains(name, "novox") && binary.BigEndian.Uint16(q[end+1:]) == 1
|
||||
if answer {
|
||||
r = append(r, 0, 1, 0, 0, 0, 0)
|
||||
} else {
|
||||
r = append(r, 0, 0, 0, 0, 0, 0)
|
||||
}
|
||||
r = append(r, question...)
|
||||
if answer {
|
||||
r = append(r, 0xC0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func udpResolver(t *testing.T, rcode byte) string {
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { pc.Close() })
|
||||
go func() {
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
n, from, err := pc.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, _ = pc.WriteTo(respond(buf[:n], rcode), from)
|
||||
}
|
||||
}()
|
||||
return pc.LocalAddr().String()
|
||||
}
|
||||
|
||||
func tcpResolver(t *testing.T) string {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { l.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := l.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var size [2]byte
|
||||
if _, err := readFull(c, size[:]); err == nil {
|
||||
q := make([]byte, binary.BigEndian.Uint16(size[:]))
|
||||
if _, err := readFull(c, q); err == nil {
|
||||
r := respond(q, 0)
|
||||
_, _ = c.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(r))), r...))
|
||||
}
|
||||
}
|
||||
c.Close()
|
||||
}
|
||||
}()
|
||||
return l.Addr().String()
|
||||
}
|
||||
|
||||
// Issue 348: the question that could not be asked during the outage — does the resolver at one address answer?
|
||||
func TestDNSAsksANamedResolverOverUDPAndTCPAndSaysItsResponseCode(t *testing.T) {
|
||||
got, err := AskDNS("novox.internal", "A", udpResolver(t, 0), false, time.Second)
|
||||
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "udp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" || got.Error != "" {
|
||||
t.Fatalf("udp: %+v, %v", got, err)
|
||||
}
|
||||
got, err = AskDNS("novox.internal", "A", tcpResolver(t), true, time.Second)
|
||||
if err != nil || got.Rcode != "NOERROR" || got.Protocol != "tcp" || len(got.Answers) != 1 || got.Answers[0] != "10.10.0.1" {
|
||||
t.Fatalf("tcp: %+v, %v", got, err)
|
||||
}
|
||||
got, _ = AskDNS("nope.internal", "A", udpResolver(t, 3), false, time.Second)
|
||||
if got.Rcode != "NXDOMAIN" || got.Error == "" || len(got.Answers) != 0 {
|
||||
t.Fatalf("no such name is a result with its code: %+v", got)
|
||||
}
|
||||
got, _ = AskDNS("novox.internal", "A", udpResolver(t, 5), false, time.Second)
|
||||
if got.Rcode != "REFUSED" || got.Error == "" {
|
||||
t.Fatalf("a refusal: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) {
|
||||
l, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
addr := l.Addr().String()
|
||||
l.Close()
|
||||
got, err := AskDNS("novox.internal", "A", addr, true, time.Second)
|
||||
if err != nil || !strings.Contains(got.Error, "connection refused") {
|
||||
t.Fatalf("nothing listening is a result naming it: %+v, %v", got, err)
|
||||
}
|
||||
if _, err := AskDNS("novox.internal", "A", "resolver.example", false, time.Second); err == nil {
|
||||
t.Fatal("a server given by name was accepted")
|
||||
}
|
||||
if _, err := AskDNS("novox.internal", "A", "", true, time.Second); err == nil {
|
||||
t.Fatal("tcp without a server was accepted")
|
||||
}
|
||||
if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" {
|
||||
t.Fatalf("default port: %s", a)
|
||||
}
|
||||
for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53",
|
||||
"[::1]:53": "[::1]:53", "::1": "[::1]:53"} {
|
||||
if a, err := serverAddress(server); err != nil || a != want {
|
||||
t.Errorf("%q: %q, %v; want %q", server, a, err, want)
|
||||
}
|
||||
}
|
||||
for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536",
|
||||
"10.10.0.1:dns", ""} {
|
||||
if a, err := serverAddress(server); err == nil {
|
||||
t.Errorf("%q was taken as %q", server, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
must := func(err error) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
must(os.MkdirAll(filepath.Join(root, "net"), 0o755))
|
||||
head := " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"
|
||||
// 127.0.0.1:53 and 10.10.0.1:53 listening, a connection on 22 (not listening), and an udp 53 bound on loopback only.
|
||||
must(os.WriteFile(filepath.Join(root, "net", "tcp"), []byte(head+
|
||||
" 0: 0100007F:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 111 1 0 100 0 0 10 0\n"+
|
||||
" 1: 01000A0A:0035 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 112 1 0 100 0 0 10 0\n"+
|
||||
" 2: 0100007F:0016 0200007F:9C40 01 00000000:00000000 00:00000000 00000000 0 0 113 1 0 100 0 0 10 0\n"), 0o644))
|
||||
must(os.WriteFile(filepath.Join(root, "net", "udp"), []byte(head+
|
||||
" 0: 0100007F:0035 00000000:0000 07 00000000:00000000 00:00000000 00000000 0 0 121 2 0 0\n"), 0o644))
|
||||
must(os.MkdirAll(filepath.Join(root, "42", "fd"), 0o755))
|
||||
must(os.WriteFile(filepath.Join(root, "42", "comm"), []byte("dnsmasq\n"), 0o644))
|
||||
must(os.Symlink("socket:[111]", filepath.Join(root, "42", "fd", "5")))
|
||||
got := Listening(root, 53)
|
||||
if got.Count != 3 || len(got.Unread) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
var where []string
|
||||
for _, s := range got.Sockets {
|
||||
where = append(where, s.Protocol+" "+s.Address)
|
||||
}
|
||||
if strings.Join(where, ",") != "tcp 10.10.0.1,tcp 127.0.0.1,udp 127.0.0.1" {
|
||||
t.Fatalf("sockets: %v", where)
|
||||
}
|
||||
if got.Sockets[1].Process != "42/dnsmasq" || got.Sockets[0].Owner == "" || !strings.Contains(got.Owners, "2 socket(s)") {
|
||||
t.Fatalf("owners: %+v", got)
|
||||
}
|
||||
if missing := Listening(t.TempDir(), 0); len(missing.Unread) != 2 {
|
||||
t.Fatalf("an unreadable table is said, not left out: %+v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListeningOnThisMachineFindsASocketItOpened(t *testing.T) {
|
||||
if _, err := os.Stat("/proc/net/tcp"); err != nil {
|
||||
t.Skip("no /proc here")
|
||||
}
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer l.Close()
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
got := Listening("/proc", port)
|
||||
if got.Count != 1 || got.Sockets[0].Address != "127.0.0.1" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if got.Sockets[0].Process == "" {
|
||||
t.Fatalf("its own socket's program is not named: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -50,8 +51,13 @@ func CheckTCP(host string, port int, timeout time.Duration) (TCPResult, error) {
|
||||
|
||||
// DNSResult is what netcheck_dns answers.
|
||||
type DNSResult struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
// Server and Protocol say which resolver was asked when one was named; empty, the machine's.
|
||||
Server string `json:"server,omitempty"`
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
// Rcode is the named resolver's response code: NOERROR, NXDOMAIN, REFUSED, SERVFAIL…
|
||||
Rcode string `json:"rcode,omitempty"`
|
||||
Answers []string `json:"answers"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Error string `json:"error,omitempty"`
|
||||
@@ -64,6 +70,13 @@ var DNSTypes = []string{"A", "AAAA", "CNAME", "TXT", "MX"}
|
||||
// the machine's /etc/resolv.conf and /etc/hosts, which is the resolver this machine's programs use.
|
||||
// A name that does not resolve is a result with its error; an unknown type is refused.
|
||||
func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
|
||||
return AskDNS(name, kind, "", false, timeout)
|
||||
}
|
||||
|
||||
// AskDNS is CheckDNS, or — with a server — the same question put to that resolver alone, over UDP or
|
||||
// TCP, its hosts file and search domains left out (hq issue 348). No answer, a refusal and a response
|
||||
// code other than NOERROR are results with their error; a malformed question is refused.
|
||||
func AskDNS(name, kind, server string, overTCP bool, timeout time.Duration) (DNSResult, error) {
|
||||
kind = strings.ToUpper(strings.TrimSpace(kind))
|
||||
if kind == "" {
|
||||
kind = "A"
|
||||
@@ -76,6 +89,37 @@ func CheckDNS(name, kind string, timeout time.Duration) (DNSResult, error) {
|
||||
return DNSResult{}, fmt.Errorf("type %q is not one netcheck_dns looks up (%s)", kind, strings.Join(DNSTypes, ", "))
|
||||
}
|
||||
out := DNSResult{Name: name, Type: kind, Answers: []string{}}
|
||||
if server != "" {
|
||||
if _, err := serverAddress(server); err != nil {
|
||||
return DNSResult{}, err
|
||||
}
|
||||
out.Server, out.Protocol = server, "udp"
|
||||
if overTCP {
|
||||
out.Protocol = "tcp"
|
||||
}
|
||||
start := time.Now()
|
||||
rcode, answers, err := askServer(server, name, kind, overTCP, timeout)
|
||||
out.ElapsedMS = time.Since(start).Milliseconds()
|
||||
if err == nil {
|
||||
out.Rcode = rcodeWords[rcode]
|
||||
if out.Rcode == "" {
|
||||
out.Rcode = "RCODE" + strconv.Itoa(rcode)
|
||||
}
|
||||
if rcode != 0 {
|
||||
out.Error = "the resolver answered " + out.Rcode
|
||||
}
|
||||
}
|
||||
out.Answers = append(out.Answers, answers...)
|
||||
if err != nil {
|
||||
out.Error = dnsWords(err, timeout)
|
||||
}
|
||||
if kind != "MX" {
|
||||
sort.Strings(out.Answers)
|
||||
}
|
||||
return out, nil
|
||||
} else if overTCP {
|
||||
return DNSResult{}, errors.New("tcp asks a named server; give server too")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
r := net.DefaultResolver
|
||||
@@ -171,3 +215,16 @@ func timeoutOf(args map[string]any) (time.Duration, error) {
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// dnsWords is a failed question in words: refused (nothing listens there), no answer in time, or what the
|
||||
// system said.
|
||||
func dnsWords(err error, timeout time.Duration) string {
|
||||
var ne net.Error
|
||||
switch {
|
||||
case errors.Is(err, syscall.ECONNREFUSED):
|
||||
return "connection refused: nothing listens on that address and port"
|
||||
case errors.As(err, &ne) && ne.Timeout():
|
||||
return fmt.Sprintf("no answer within %s", timeout)
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
@@ -57,12 +57,12 @@ func TestTimeoutIsDefaultedAndBounded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBothToolsAreListedUnprefixed(t *testing.T) {
|
||||
func TestTheToolsAreListedUnprefixed(t *testing.T) {
|
||||
names := map[string]bool{}
|
||||
for _, tool := range tools() {
|
||||
names[tool.Name] = true
|
||||
}
|
||||
if !names["netcheck_tcp"] || !names["netcheck_dns"] || len(names) != 2 {
|
||||
if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 {
|
||||
t.Fatalf("tools: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package main
|
||||
|
||||
// What listens on this machine (hq issue 348): the kernel's own socket tables, read where `ss -lunt` reads
|
||||
// them. During the 2026-10-09 outage the resolver's unit stayed active while it no longer listened on its
|
||||
// private address, and nothing on the mesh could show that: whether a socket is bound is the kernel's to
|
||||
// say, not the unit's.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Socket is one listening TCP socket or one bound UDP socket.
|
||||
type Socket struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
// Process is the program holding it, "pid/name"; empty when Owner says why it is not known.
|
||||
Process string `json:"process,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
inode string
|
||||
}
|
||||
|
||||
// ListeningResult is what netcheck_listening answers.
|
||||
type ListeningResult struct {
|
||||
Sockets []Socket `json:"sockets"`
|
||||
Count int `json:"count"`
|
||||
// Unread names a table that could not be read: a socket in it is missing from the list, not absent.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Owners says how many sockets' programs are unknown and why, when any is.
|
||||
Owners string `json:"owners,omitempty"`
|
||||
}
|
||||
|
||||
// Listening reads the socket tables under root (the machine's /proc), keeps TCP sockets in the listen
|
||||
// state and every bound UDP socket, on port when port is not 0.
|
||||
func Listening(root string, port int) ListeningResult {
|
||||
out := ListeningResult{Sockets: []Socket{}}
|
||||
for _, t := range []struct{ file, protocol string }{
|
||||
{"tcp", "tcp"}, {"tcp6", "tcp6"}, {"udp", "udp"}, {"udp6", "udp6"},
|
||||
} {
|
||||
f, err := os.Open(filepath.Join(root, "net", t.file))
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) || !strings.HasSuffix(t.file, "6") {
|
||||
out.Unread = append(out.Unread, t.file+": "+err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
sockets, err := parseSocketTable(f, t.protocol, port)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
out.Unread = append(out.Unread, t.file+": "+err.Error())
|
||||
}
|
||||
out.Sockets = append(out.Sockets, sockets...)
|
||||
}
|
||||
unknown := owners(root, out.Sockets)
|
||||
if unknown > 0 {
|
||||
out.Owners = fmt.Sprintf("%d socket(s) without their program: held by a process of another account, whose "+
|
||||
"open files this tool may not read; the address and port are the kernel's and complete", unknown)
|
||||
}
|
||||
sort.Slice(out.Sockets, func(i, j int) bool {
|
||||
a, b := out.Sockets[i], out.Sockets[j]
|
||||
if a.Port != b.Port {
|
||||
return a.Port < b.Port
|
||||
}
|
||||
if a.Protocol != b.Protocol {
|
||||
return a.Protocol < b.Protocol
|
||||
}
|
||||
return a.Address < b.Address
|
||||
})
|
||||
out.Count = len(out.Sockets)
|
||||
return out
|
||||
}
|
||||
|
||||
// parseSocketTable reads one /proc/net table: "sl local_address rem_address st … inode".
|
||||
func parseSocketTable(f *os.File, protocol string, port int) ([]Socket, error) {
|
||||
var out []Socket
|
||||
sc := bufio.NewScanner(f)
|
||||
first := true
|
||||
for sc.Scan() {
|
||||
if first {
|
||||
first = false
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) < 10 {
|
||||
continue
|
||||
}
|
||||
state := fields[3]
|
||||
tcp := strings.HasPrefix(protocol, "tcp")
|
||||
// TCP 0A is LISTEN; UDP 07 is a bound socket (unconnected), 01 a connected one, which is not listening.
|
||||
if (tcp && state != "0A") || (!tcp && state != "07") {
|
||||
continue
|
||||
}
|
||||
addr, p, err := hexAddress(fields[1])
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if port != 0 && p != port {
|
||||
continue
|
||||
}
|
||||
out = append(out, Socket{Protocol: protocol, Address: addr, Port: p, inode: fields[9]})
|
||||
}
|
||||
return out, sc.Err()
|
||||
}
|
||||
|
||||
// hexAddress reads the kernel's "0100007F:0035": the address in host order per 32-bit word, the port big-endian.
|
||||
func hexAddress(s string) (string, int, error) {
|
||||
a, p, ok := strings.Cut(s, ":")
|
||||
if !ok {
|
||||
return "", 0, fmt.Errorf("address %q", s)
|
||||
}
|
||||
port, err := strconv.ParseUint(p, 16, 16)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("port %q", p)
|
||||
}
|
||||
raw, err := hex.DecodeString(a)
|
||||
if err != nil || (len(raw) != 4 && len(raw) != 16) {
|
||||
return "", 0, fmt.Errorf("address %q", a)
|
||||
}
|
||||
ip := make(net.IP, len(raw))
|
||||
for w := 0; w < len(raw); w += 4 {
|
||||
ip[w], ip[w+1], ip[w+2], ip[w+3] = raw[w+3], raw[w+2], raw[w+1], raw[w]
|
||||
}
|
||||
return ip.String(), int(port), nil
|
||||
}
|
||||
|
||||
// owners names each socket's program from the processes' open files, as far as they can be read, and
|
||||
// counts those it could not name.
|
||||
func owners(root string, sockets []Socket) int {
|
||||
want := map[string][]int{}
|
||||
for i, s := range sockets {
|
||||
want["socket:["+s.inode+"]"] = append(want["socket:["+s.inode+"]"], i)
|
||||
}
|
||||
if len(want) == 0 {
|
||||
return 0
|
||||
}
|
||||
procs, _ := os.ReadDir(root)
|
||||
for _, p := range procs {
|
||||
pid, err := strconv.Atoi(p.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fds, err := os.ReadDir(filepath.Join(root, p.Name(), "fd"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, fd := range fds {
|
||||
link, err := os.Readlink(filepath.Join(root, p.Name(), "fd", fd.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
idx, ok := want[link]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, _ := os.ReadFile(filepath.Join(root, p.Name(), "comm"))
|
||||
for _, i := range idx {
|
||||
if sockets[i].Process == "" {
|
||||
sockets[i].Process = fmt.Sprintf("%d/%s", pid, strings.TrimSpace(string(name)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
unknown := 0
|
||||
for i := range sockets {
|
||||
if sockets[i].Process == "" {
|
||||
sockets[i].Owner = "not readable by this tool's account"
|
||||
unknown++
|
||||
}
|
||||
}
|
||||
return unknown
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
// netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches
|
||||
// and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's
|
||||
// own sockets and resolver — a TCP connect and a DNS lookup — and nothing that changes anything.
|
||||
// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named),
|
||||
// and what listens here — and nothing that changes anything.
|
||||
// The module's HTTP check is its TypeScript bundle; the runtime serves both under one module.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
@@ -21,6 +23,29 @@ func main() {
|
||||
|
||||
func tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{
|
||||
Name: "netcheck_listening",
|
||||
Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " +
|
||||
"from the kernel's own tables (what ss -lunt reads), with the program holding it where this tool may see it. " +
|
||||
"port narrows it to one port, e.g. 53 to see on which addresses the resolver answers.",
|
||||
Input: map[string]any{
|
||||
"port": map[string]any{"type": "integer", "description": "only this port (default every port)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
port := 0
|
||||
if _, given := args["port"]; given {
|
||||
p, err := whole(args, "port", 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p < 1 || p > 65535 {
|
||||
return nil, fmt.Errorf("port %d is not a port (1-65535)", p)
|
||||
}
|
||||
port = p
|
||||
}
|
||||
return Listening("/proc", port), nil
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "netcheck_tcp",
|
||||
Description: "Check whether a TCP port is reachable from this machine: opens one connection " +
|
||||
@@ -48,11 +73,14 @@ func tools() []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "netcheck_dns",
|
||||
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts). " +
|
||||
Description: "Look a name up with this machine's resolver (its /etc/resolv.conf and /etc/hosts), or — with server — ask " +
|
||||
"that one resolver alone over UDP (or TCP), answering its response code (NOERROR, NXDOMAIN, REFUSED…). " +
|
||||
"type is A, AAAA, CNAME, TXT or MX; answers the records found, or the error.",
|
||||
Input: map[string]any{
|
||||
"name": map[string]any{"type": "string", "description": "the name to look up"},
|
||||
"type": map[string]any{"type": "string", "enum": []string{"A", "AAAA", "CNAME", "TXT", "MX"}, "description": "record type (default A)"},
|
||||
"server": map[string]any{"type": "string", "description": "ask this resolver alone: an address, with :port or without (53), e.g. 10.10.0.1"},
|
||||
"tcp": map[string]any{"type": "boolean", "description": "ask the server over TCP rather than UDP (default false)"},
|
||||
"timeout_ms": map[string]any{"type": "integer", "description": "give up after this long (default 3000, at most 30000)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -65,7 +93,9 @@ func tools() []stdio.Tool {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return CheckDNS(name, kind, timeout)
|
||||
server, _ := args["server"].(string)
|
||||
overTCP, _ := args["tcp"].(bool)
|
||||
return AskDNS(name, kind, strings.TrimSpace(server), overTCP, timeout)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -4,8 +4,20 @@
|
||||
"tools": [
|
||||
"netcheck_tcp",
|
||||
"netcheck_dns",
|
||||
"netcheck_http"
|
||||
"netcheck_http",
|
||||
"netcheck_listening"
|
||||
],
|
||||
"replaces": {
|
||||
"netcheck_listening": [
|
||||
"ss -lunt",
|
||||
"ss -ltn",
|
||||
"netstat -lnt"
|
||||
],
|
||||
"netcheck_dns": [
|
||||
"dig",
|
||||
"nslookup"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user