Judge the machine's own networking beside what its modules run (hq ADR 0241)
A VPN client rewrote the laptop's resolver file and every mesh name failed while each module read healthy: nothing asked the machine. The engine now looks every 30 s at the resolver file the uplink holder declared (naming the program that rewrote it), the names through each listed resolver (NXDOMAIN for a mesh name's IPv6 address is a finding, issue 262), the tunnel's handshake with the hub, the bus and the default route; a part is unhealthy on its second failing look, and the statement carries it.
This commit is contained in:
+105
-4
@@ -17,6 +17,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
@@ -35,6 +36,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/inventory"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/network"
|
||||
"github.com/novox/mesh-host/internal/outward"
|
||||
"github.com/novox/mesh-host/internal/profile"
|
||||
"github.com/novox/mesh-host/internal/reachable"
|
||||
@@ -1168,6 +1170,10 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// spaced to the budget (ADR 0240 Phase B); a tool is asked of this machine's node tools over the
|
||||
// link open at the time.
|
||||
judging.Probes = &liveness.Probes{AskTool: queue.AskTool}
|
||||
// And the machine's own networking (novox/hq ADR 0241): the resolver file the uplink holder
|
||||
// declared, the names through it, the tunnel, the bus and the route — said in the same statement.
|
||||
netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
|
||||
hostOf(mine.Membership.Broker)))
|
||||
go judging.Probe(aside)
|
||||
go judgeWhatRuns(aside, judging, queue, say)
|
||||
}
|
||||
@@ -1361,6 +1367,74 @@ const ReconcileEvery = 5 * time.Minute
|
||||
// reports no health, and in a test.
|
||||
var judging *liveness.Judge
|
||||
|
||||
// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a
|
||||
// one-shot command and in a test, which say nothing of the network.
|
||||
var netJudge networkJudge
|
||||
|
||||
type networkJudge struct {
|
||||
mu sync.Mutex
|
||||
m *network.Judge
|
||||
// The resolver file the last apply declared, kept for a judge made after it.
|
||||
content, owner string
|
||||
declared bool
|
||||
}
|
||||
|
||||
func (n *networkJudge) get() *network.Judge {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
return n.m
|
||||
}
|
||||
|
||||
func (n *networkJudge) set(m *network.Judge) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
n.m = m
|
||||
m.Declare(n.content, n.owner, n.declared)
|
||||
}
|
||||
|
||||
// declare is the resolver file an apply just applied, for the judge now and any made later.
|
||||
func (n *networkJudge) declare(content, owner string, ok bool) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
n.content, n.owner, n.declared = content, owner, ok
|
||||
if n.m != nil {
|
||||
n.m.Declare(content, owner, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// hostOf is the bus's name without its port: the mesh name the machine needs most. Empty when the bus
|
||||
// is reached by address, and then no mesh name is asked.
|
||||
func hostOf(broker string) string {
|
||||
host := broker
|
||||
if h, _, err := net.SplitHostPort(broker); err == nil {
|
||||
host = h
|
||||
}
|
||||
if net.ParseIP(host) != nil {
|
||||
return ""
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// declaredResolvConf is the resolver file a declaration has a module write whole — the uplink holder's
|
||||
// (ADR 0223) — and that module.
|
||||
func declaredResolvConf(d *declaration.Declaration) (string, string, bool) {
|
||||
if d == nil {
|
||||
return "", "", false
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" {
|
||||
continue
|
||||
}
|
||||
module, ok := liveness.ModuleOf(f.ID)
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
return f.Content, module, true
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// How often a statement of health is said between reports: again every minute while anything is not
|
||||
// healthy (to-be 48 §4), so a lost event is not a lost fault; and every five minutes anyway, so a
|
||||
// controller that restarted knows a healthy machine's state without waiting for its next apply.
|
||||
@@ -1387,6 +1461,19 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
st, changed := j.Look(ctx)
|
||||
owed = owed || changed
|
||||
var netSt *network.Statement
|
||||
if n := netJudge.get(); n != nil {
|
||||
ns, netChanged := n.Look(ctx)
|
||||
netSt = &ns
|
||||
owed = owed || netChanged
|
||||
if netChanged {
|
||||
for _, p := range ns.Parts {
|
||||
if p.State == network.Unhealthy {
|
||||
say(fmt.Sprintf("this machine's network is unhealthy: %s: %s (%s)", p.Part, p.Reason, p.Said))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Never more looks than the budget (ADR 0240): said when the engine has to space its own out.
|
||||
if sp := j.Spacing(); sp != spaced {
|
||||
if sp > 1 {
|
||||
@@ -1396,7 +1483,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
spaced = sp
|
||||
}
|
||||
since := time.Since(lastSaid)
|
||||
if !owed && !(!st.Healthy() && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy)
|
||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
continue
|
||||
}
|
||||
if changed {
|
||||
@@ -1407,14 +1495,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *healthAsReported(st)) {
|
||||
if queue.SayHealth(ctx, *healthAsReported(st, netSt)) {
|
||||
lastSaid, owed = time.Now(), false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healthAsReported is a statement as the report and the event carry it.
|
||||
func healthAsReported(st liveness.Statement) *link.Health {
|
||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||
// ReadinessContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase
|
||||
// B), which is what tells the controller it may be sent the field.
|
||||
h := &link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
@@ -1423,6 +1511,13 @@ func healthAsReported(st liveness.Statement) *link.Health {
|
||||
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
|
||||
Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf()})
|
||||
}
|
||||
if ns != nil && ns.State != "" {
|
||||
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
|
||||
for _, p := range ns.Parts {
|
||||
h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
|
||||
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: p.Streak})
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
@@ -1657,7 +1752,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
if j := judging; j != nil {
|
||||
j.Set(liveness.LongRunning(declared, held))
|
||||
st, _ := j.Look(ctx)
|
||||
report.Health = healthAsReported(st)
|
||||
netJudge.declare(declaredResolvConf(declared))
|
||||
var netSt *network.Statement
|
||||
if n := netJudge.get(); n != nil {
|
||||
ns := n.Last()
|
||||
netSt = &ns
|
||||
}
|
||||
report.Health = healthAsReported(st, netSt)
|
||||
}
|
||||
// Which of this machine's links face outside, for the filter the mesh writes around them
|
||||
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/network"
|
||||
)
|
||||
|
||||
// The machine's networking in the engine's statement (novox/hq ADR 0241): the file judged is the one the
|
||||
// uplink holder declares whole, the mesh name asked is the bus's, and the statement carries the parts.
|
||||
|
||||
func TestTheResolverFileJudgedIsTheOneAModuleDeclaresWhole(t *testing.T) {
|
||||
d := &declaration.Declaration{Resources: []declaration.Resource{
|
||||
&declaration.File{ID: "hosts.file", Type: "file", Path: "/etc/hosts", Content: "x"},
|
||||
&declaration.File{ID: "networkmanager.resolv", Type: "file", Path: network.ResolvConf, Content: "nameserver 10.10.0.1\n"},
|
||||
}}
|
||||
content, owner, ok := declaredResolvConf(d)
|
||||
if !ok || owner != "networkmanager" || content != "nameserver 10.10.0.1\n" {
|
||||
t.Fatalf("got %q %q %v", content, owner, ok)
|
||||
}
|
||||
d.Resources[1].(*declaration.File).CreateOnce = true
|
||||
if _, _, ok := declaredResolvConf(d); ok {
|
||||
t.Fatal("a seed nobody holds the machine to was judged as the declared file")
|
||||
}
|
||||
if _, _, ok := declaredResolvConf(nil); ok {
|
||||
t.Fatal("no declaration declared a file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMeshNameAskedIsTheBuses(t *testing.T) {
|
||||
for broker, want := range map[string]string{"anchor.internal:4222": "anchor.internal", "192.0.2.10:5671": "",
|
||||
"anchor.internal": "anchor.internal", "[2001:db8::1]:4222": ""} {
|
||||
if got := hostOf(broker); got != want {
|
||||
t.Errorf("%s: got %q, want %q", broker, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheStatementCarriesTheNetwork(t *testing.T) {
|
||||
at := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||
h := healthAsReported(liveness.Statement{At: at}, &network.Statement{State: network.Unhealthy, Since: at,
|
||||
Parts: []network.Part{{Part: network.PartResolvConf, State: network.Unhealthy, Writer: "FortiClient",
|
||||
Owner: "networkmanager", Reason: "the resolver file was rewritten by another program", Since: at}}})
|
||||
if h.Network == nil || h.Network.State != network.Unhealthy || len(h.Network.Parts) != 1 ||
|
||||
h.Network.Parts[0].Writer != "FortiClient" || h.Network.Parts[0].Owner != "networkmanager" {
|
||||
t.Fatalf("the statement says %+v", h.Network)
|
||||
}
|
||||
if h := healthAsReported(liveness.Statement{At: at}, nil); h.Network != nil {
|
||||
t.Fatal("an engine with no network judge said a network")
|
||||
}
|
||||
}
|
||||
@@ -238,6 +238,36 @@ type Health struct {
|
||||
At time.Time `json:"at"`
|
||||
// Resources are every long-running resource of a module this machine runs, by module and id.
|
||||
Resources []ResourceHealth `json:"resources"`
|
||||
// Network is the machine's own networking, judged by its engine (novox/hq ADR 0241): its resolver
|
||||
// file, its names, its tunnel, its bus and its route. Absent from an engine older than that judging,
|
||||
// which the controller reads as "not known", never as healthy.
|
||||
Network *NetworkHealth `json:"network,omitempty"`
|
||||
}
|
||||
|
||||
// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since
|
||||
// when, and each part.
|
||||
type NetworkHealth struct {
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
Parts []NetworkPart `json:"parts"`
|
||||
}
|
||||
|
||||
// NetworkPart is one part of a machine's networking, as its engine judged it on its second look.
|
||||
type NetworkPart struct {
|
||||
// Part is resolv-conf, names, tunnel, bus or route.
|
||||
Part string `json:"part"`
|
||||
State string `json:"state"`
|
||||
// Reason is why it is not healthy, in words with no address, path or domain; Said the detail.
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
// Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose
|
||||
// file it is — the uplink's holder.
|
||||
Writer string `json:"writer,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
// Toward is what the failure points at: "hub", or each resolver's address that failed.
|
||||
Toward []string `json:"toward,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Streak int `json:"streak,omitempty"`
|
||||
}
|
||||
|
||||
// The states a long-running resource is said in (ADR 0240 §4).
|
||||
|
||||
@@ -479,6 +479,15 @@ func declaredIn(body []byte) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the
|
||||
// machine's networking its engine judges).
|
||||
func (q *Queue) Linked() bool {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return q.bus != nil
|
||||
}
|
||||
|
||||
// SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether
|
||||
// the bus took it. **Not through the worker**: a statement is a word about the machine as it is, not an
|
||||
// account of an apply, and it must not wait behind one — a container crash-looping while a long apply
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package network
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The record types a look asks for.
|
||||
const (
|
||||
TypeA uint16 = 1
|
||||
TypeAAAA uint16 = 28
|
||||
)
|
||||
|
||||
// The answers a resolver gives that a look tells apart.
|
||||
const (
|
||||
RcodeOK = 0
|
||||
RcodeServFail = 2
|
||||
RcodeNXDomain = 3
|
||||
RcodeRefused = 5
|
||||
)
|
||||
|
||||
// Answer is what one resolver said to one question: its code, how many records of the type asked it
|
||||
// gave, and how long it took. A question with no answer at all is an error, never an Answer.
|
||||
type Answer struct {
|
||||
Rcode int
|
||||
Records int
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// Ask asks one resolver one question over UDP, and reads its code and how many records of the type
|
||||
// asked it answered with. **It tells "no such name" from "no record of that type"** — the C library's
|
||||
// lookup does not, and that difference is issue 262: an Alpine container failed a mesh name because a
|
||||
// resolver said NXDOMAIN for its IPv6 address where it should have said there was none.
|
||||
func Ask(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (Answer, error) {
|
||||
start := time.Now()
|
||||
query, id, err := question(name, qtype)
|
||||
if err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
if net.ParseIP(server) != nil {
|
||||
server = net.JoinHostPort(server, "53")
|
||||
}
|
||||
d := net.Dialer{Timeout: timeout}
|
||||
conn, err := d.DialContext(ctx, "udp", server)
|
||||
if err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
defer conn.Close()
|
||||
_ = conn.SetDeadline(start.Add(timeout))
|
||||
if _, err := conn.Write(query); err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil {
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
return Answer{}, fmt.Errorf("no answer within %s", timeout)
|
||||
}
|
||||
return Answer{}, err
|
||||
}
|
||||
a, ours, err := parse(buf[:n], id, qtype)
|
||||
if !ours {
|
||||
continue // a late answer to somebody else's question on this port
|
||||
}
|
||||
if err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
a.Took = time.Since(start)
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// question is one query: a header asking for recursion, and the name and type.
|
||||
func question(name string, qtype uint16) ([]byte, uint16, error) {
|
||||
var idb [2]byte
|
||||
if _, err := rand.Read(idb[:]); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
id := binary.BigEndian.Uint16(idb[:])
|
||||
msg := make([]byte, 12, 64)
|
||||
binary.BigEndian.PutUint16(msg[0:], id)
|
||||
msg[2] = 0x01 // recursion desired
|
||||
binary.BigEndian.PutUint16(msg[4:], 1)
|
||||
for _, label := range strings.Split(strings.TrimSuffix(name, "."), ".") {
|
||||
if label == "" || len(label) > 63 {
|
||||
return nil, 0, fmt.Errorf("%q is not a name that can be asked", name)
|
||||
}
|
||||
msg = append(msg, byte(len(label)))
|
||||
msg = append(msg, label...)
|
||||
}
|
||||
msg = append(msg, 0, byte(qtype>>8), byte(qtype), 0, 1)
|
||||
return msg, id, nil
|
||||
}
|
||||
|
||||
// parse reads an answer: false when it is not the answer to this question.
|
||||
func parse(msg []byte, id, qtype uint16) (Answer, bool, error) {
|
||||
if len(msg) < 12 || binary.BigEndian.Uint16(msg[0:]) != id || msg[2]&0x80 == 0 {
|
||||
return Answer{}, false, nil
|
||||
}
|
||||
a := Answer{Rcode: int(msg[3] & 0x0f)}
|
||||
qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:]))
|
||||
at := 12
|
||||
for i := 0; i < qd; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return a, true, err
|
||||
}
|
||||
at += 4
|
||||
}
|
||||
for i := 0; i < an; i++ {
|
||||
var err error
|
||||
if at, err = skipName(msg, at); err != nil {
|
||||
return a, true, err
|
||||
}
|
||||
if at+10 > len(msg) {
|
||||
return a, true, errors.New("the answer is cut short")
|
||||
}
|
||||
typ := binary.BigEndian.Uint16(msg[at:])
|
||||
length := int(binary.BigEndian.Uint16(msg[at+8:]))
|
||||
at += 10 + length
|
||||
if at > len(msg) {
|
||||
return a, true, errors.New("the answer is cut short")
|
||||
}
|
||||
if typ == qtype {
|
||||
a.Records++
|
||||
}
|
||||
}
|
||||
return a, true, nil
|
||||
}
|
||||
|
||||
// skipName steps over a name, compressed or not.
|
||||
func skipName(msg []byte, at int) (int, error) {
|
||||
for {
|
||||
if at >= len(msg) {
|
||||
return 0, errors.New("the answer is cut short")
|
||||
}
|
||||
l := int(msg[at])
|
||||
switch {
|
||||
case l == 0:
|
||||
return at + 1, nil
|
||||
case l&0xc0 == 0xc0:
|
||||
return at + 2, nil
|
||||
default:
|
||||
at += 1 + l
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,598 @@
|
||||
// Package network is the node-engine judging its own machine's networking (novox/hq ADR 0241, which
|
||||
// extends ADR 0240 from what a module runs to the machine it runs on).
|
||||
//
|
||||
// **A machine whose names stopped resolving read healthy.** On the laptop a corporate VPN client rewrote
|
||||
// `/etc/resolv.conf` when it connected, replacing the mesh's resolvers (ADR 0223) with its own: mesh names
|
||||
// failed, sometimes public ones too, and agents saw "no such host" for the services they call. The
|
||||
// node-engine wrote the file back at its next reconcile, the client rewrote it again, and nothing said so —
|
||||
// every module's own check was green, because no check asked the machine. A resolver slow under load
|
||||
// (issue 277) and the tunnel to the hub are the same kind of fact: about the machine, under every module.
|
||||
//
|
||||
// Every LookEvery the judge looks at five parts, each cheaply:
|
||||
//
|
||||
// - **resolv-conf**: the file is what the uplink holder declared (ADR 0117, ADR 0223). Rewritten by
|
||||
// another program, it says so — naming the program where the file, its link or what runs shows it;
|
||||
// - **names**: every resolver the file lists answers a mesh name with an address and its IPv6 question
|
||||
// with "none" rather than "no such name" (issue 262), and a public name with an address, within the
|
||||
// time the file itself tells the C library to wait;
|
||||
// - **tunnel**: the mesh's interface has a fresh handshake with the hub — on the hub, with any machine;
|
||||
// - **bus**: the link to the bus is open;
|
||||
// - **route**: the machine has a default route.
|
||||
//
|
||||
// **The two-look rule** (issue 277): a part is said unhealthy on its second failing look in a row, and
|
||||
// healthy again on its first passing one. One unanswered datagram is not a finding.
|
||||
//
|
||||
// **It reads; it never acts** (ADR 0240 rule 6): nothing here writes the file back, restarts a link or
|
||||
// asks a reconcile. The reconcile holds the file as it always has; this says when somebody else holds it.
|
||||
package network
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The bounds.
|
||||
const (
|
||||
// LookEvery is how often the parts are looked at: about six datagrams per resolver and a read of
|
||||
// three small files a minute, two looks to a finding inside the gate's first judging.
|
||||
LookEvery = 30 * time.Second
|
||||
// Confirm is how many failing looks in a row make a part unhealthy (issue 277).
|
||||
Confirm = 2
|
||||
// StaleHandshake is how old the newest handshake with the hub may be. WireGuard renews a session
|
||||
// every two minutes while anything passes over it, and the bus pings every two: past five, nothing
|
||||
// has passed over the tunnel.
|
||||
StaleHandshake = 5 * time.Minute
|
||||
// ResolvConf is the file the uplink holder writes.
|
||||
ResolvConf = "/etc/resolv.conf"
|
||||
// PublicName is the public name asked: reserved for exactly this kind of use, answered by every
|
||||
// public resolver, and belonging to no installation.
|
||||
PublicName = "example.com"
|
||||
// Interface is the mesh's own tunnel.
|
||||
Interface = "mesh0"
|
||||
)
|
||||
|
||||
// The parts.
|
||||
const (
|
||||
PartResolvConf = "resolv-conf"
|
||||
PartNames = "names"
|
||||
PartTunnel = "tunnel"
|
||||
PartBus = "bus"
|
||||
PartRoute = "route"
|
||||
)
|
||||
|
||||
// Parts is every part, in the order a person reads them.
|
||||
var Parts = []string{PartResolvConf, PartNames, PartTunnel, PartBus, PartRoute}
|
||||
|
||||
// The states, the words liveness says them in.
|
||||
const (
|
||||
Healthy = "healthy"
|
||||
Unhealthy = "unhealthy"
|
||||
Unknown = "unknown"
|
||||
)
|
||||
|
||||
// TowardHub is what a part that fails toward the hub names: the tunnel and the bus.
|
||||
const TowardHub = "hub"
|
||||
|
||||
// Finding is one look at one part.
|
||||
type Finding struct {
|
||||
// Skip says the part is not judged on this machine: nothing declares the file, there is no tunnel.
|
||||
Skip bool
|
||||
OK bool
|
||||
// Reason is why it is not healthy, in words that carry no address, path or name with its domain:
|
||||
// it may reach the operator's channel. Said is the detail, which stays inside the mesh.
|
||||
Reason string
|
||||
Said string
|
||||
// Writer is the program that rewrote the file, when the file, its link or what runs shows it.
|
||||
Writer string
|
||||
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
|
||||
Toward []string
|
||||
}
|
||||
|
||||
// Part is one part's state, as the statement says it.
|
||||
type Part struct {
|
||||
Part string `json:"part"`
|
||||
State string `json:"state"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Said string `json:"said,omitempty"`
|
||||
Writer string `json:"writer,omitempty"`
|
||||
Owner string `json:"owner,omitempty"`
|
||||
Toward []string `json:"toward,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Streak int `json:"streak,omitempty"`
|
||||
}
|
||||
|
||||
// Statement is one look at the machine's networking: the worst of its parts, since when, and each part.
|
||||
type Statement struct {
|
||||
State string `json:"state"`
|
||||
Since time.Time `json:"since"`
|
||||
At time.Time `json:"at"`
|
||||
Parts []Part `json:"parts"`
|
||||
}
|
||||
|
||||
// Machine is what a look reads, replaced in tests.
|
||||
type Machine struct {
|
||||
// ResolvPath and ProcNet are where the file and the routing tables are.
|
||||
ResolvPath string
|
||||
ProcNet string
|
||||
// Ask asks one resolver one question.
|
||||
Ask func(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (Answer, error)
|
||||
// Run runs a command: `wg`, to read the tunnel.
|
||||
Run func(ctx context.Context, name string, args ...string) (string, error)
|
||||
// Linked says whether the link to the bus is open now.
|
||||
Linked func() bool
|
||||
// Running is the names of the programs running, to name a writer by. Nil reads /proc.
|
||||
Running func() []string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// declared is the file as the uplink holder declared it.
|
||||
type declared struct {
|
||||
content string
|
||||
owner string
|
||||
}
|
||||
|
||||
type kept struct {
|
||||
state string
|
||||
since time.Time
|
||||
streak int
|
||||
last Finding
|
||||
}
|
||||
|
||||
// Judge is the one judge of this machine's networking. Safe for the apply and the looking loop at once.
|
||||
type Judge struct {
|
||||
m Machine
|
||||
// MeshName is a name only the mesh's resolvers answer: the bus's own, which this machine needs most.
|
||||
MeshName string
|
||||
|
||||
mu sync.Mutex
|
||||
file *declared
|
||||
kept map[string]*kept
|
||||
said Statement
|
||||
lookedAt time.Time
|
||||
overall kept
|
||||
}
|
||||
|
||||
// New is a judge of this machine, asking meshName as the mesh's name (empty when the bus is reached by
|
||||
// address, and then no mesh name is asked).
|
||||
func New(m Machine, meshName string) *Judge {
|
||||
if m.ResolvPath == "" {
|
||||
m.ResolvPath = ResolvConf
|
||||
}
|
||||
if m.ProcNet == "" {
|
||||
m.ProcNet = "/proc/net"
|
||||
}
|
||||
if m.Ask == nil {
|
||||
m.Ask = Ask
|
||||
}
|
||||
if m.Running == nil {
|
||||
m.Running = running
|
||||
}
|
||||
if m.Now == nil {
|
||||
m.Now = time.Now
|
||||
}
|
||||
return &Judge{m: m, MeshName: meshName, kept: map[string]*kept{}}
|
||||
}
|
||||
|
||||
// Declare is the file the uplink holder declared, from the declaration the apply just applied, and the
|
||||
// module that declared it; ok false when nothing declares it whole, and then the file is not judged.
|
||||
func (j *Judge) Declare(content, owner string, ok bool) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
if !ok {
|
||||
j.file = nil
|
||||
return
|
||||
}
|
||||
j.file = &declared{content: content, owner: owner}
|
||||
}
|
||||
|
||||
// Look looks again when a look is due, and answers the statement and whether anything changed since the
|
||||
// last; between looks it answers the last statement, unchanged.
|
||||
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
now := j.m.Now()
|
||||
if !j.lookedAt.IsZero() && now.Sub(j.lookedAt) < LookEvery {
|
||||
return j.said, false
|
||||
}
|
||||
j.lookedAt = now
|
||||
findings := map[string]Finding{
|
||||
PartResolvConf: j.lookFile(),
|
||||
PartNames: j.lookNames(ctx),
|
||||
PartTunnel: j.lookTunnel(ctx, now),
|
||||
PartBus: j.lookBus(),
|
||||
PartRoute: j.lookRoute(),
|
||||
}
|
||||
st := Statement{At: now, Parts: []Part{}}
|
||||
changed := false
|
||||
worst := Healthy
|
||||
for _, name := range Parts {
|
||||
f := findings[name]
|
||||
k := j.kept[name]
|
||||
if f.Skip {
|
||||
if k != nil {
|
||||
delete(j.kept, name)
|
||||
changed = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if k == nil {
|
||||
k = &kept{state: Unknown}
|
||||
j.kept[name] = k
|
||||
}
|
||||
before := k.state
|
||||
if f.OK {
|
||||
k.streak = 0
|
||||
if k.state != Healthy {
|
||||
k.state, k.since = Healthy, now
|
||||
}
|
||||
} else {
|
||||
k.streak++
|
||||
if k.streak >= Confirm && k.state != Unhealthy {
|
||||
k.state, k.since = Unhealthy, now
|
||||
}
|
||||
}
|
||||
k.last = f
|
||||
changed = changed || before != k.state
|
||||
p := Part{Part: name, State: k.state, Since: k.since, Streak: k.streak}
|
||||
if k.state == Unhealthy {
|
||||
p.Reason, p.Said, p.Writer, p.Toward = f.Reason, f.Said, f.Writer, f.Toward
|
||||
if name == PartResolvConf && j.file != nil {
|
||||
p.Owner = j.file.owner
|
||||
}
|
||||
}
|
||||
if k.state == Unknown && k.streak > 0 {
|
||||
// Failing once: not yet a finding, and said as not known rather than as healthy.
|
||||
p.Reason = "one look failed; a second decides"
|
||||
}
|
||||
st.Parts = append(st.Parts, p)
|
||||
switch {
|
||||
case k.state == Unhealthy:
|
||||
worst = Unhealthy
|
||||
case k.state == Unknown && worst == Healthy:
|
||||
worst = Unknown
|
||||
}
|
||||
}
|
||||
if j.overall.state != worst || j.overall.since.IsZero() {
|
||||
j.overall.state, j.overall.since = worst, now
|
||||
changed = true
|
||||
}
|
||||
st.State, st.Since = worst, j.overall.since
|
||||
j.said = st
|
||||
return st, changed
|
||||
}
|
||||
|
||||
// Last is the statement said last, without looking.
|
||||
func (j *Judge) Last() Statement {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.said
|
||||
}
|
||||
|
||||
// lookFile compares the file with what the uplink holder declared.
|
||||
func (j *Judge) lookFile() Finding {
|
||||
if j.file == nil {
|
||||
return Finding{Skip: true}
|
||||
}
|
||||
path := j.m.ResolvPath
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return Finding{Reason: "the resolver file is missing", Said: err.Error(), Toward: nil}
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
target, _ := os.Readlink(path)
|
||||
return Finding{Reason: "the resolver file was replaced by a link, so another program now writes it",
|
||||
Said: fmt.Sprintf("%s is a link to %s, not the file %s declares", path, target, j.file.owner),
|
||||
Writer: writerOfLink(target)}
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Finding{Reason: "the resolver file cannot be read", Said: err.Error()}
|
||||
}
|
||||
if strings.TrimSpace(string(raw)) == strings.TrimSpace(j.file.content) {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
writer, why := j.writerOf(string(raw), info.ModTime())
|
||||
said := fmt.Sprintf("%s differs from what %s declares: it lists %s where %s is declared; changed %s",
|
||||
path, j.file.owner, listOrNone(nameservers(string(raw))), listOrNone(nameservers(j.file.content)),
|
||||
info.ModTime().UTC().Format(time.RFC3339))
|
||||
if why != "" {
|
||||
said += "; " + why
|
||||
}
|
||||
return Finding{Reason: "the resolver file was rewritten by another program", Said: said, Writer: writer}
|
||||
}
|
||||
|
||||
// lookNames asks every resolver the file lists — as the machine's programs read it now, whoever wrote it.
|
||||
func (j *Judge) lookNames(ctx context.Context) Finding {
|
||||
raw, err := os.ReadFile(j.m.ResolvPath)
|
||||
if err != nil {
|
||||
return Finding{Reason: "no resolver can be read from the resolver file", Said: err.Error()}
|
||||
}
|
||||
servers := nameservers(string(raw))
|
||||
if len(servers) == 0 {
|
||||
return Finding{Reason: "the resolver file lists no resolver", Said: j.m.ResolvPath + " lists no nameserver"}
|
||||
}
|
||||
if len(servers) > 3 {
|
||||
servers = servers[:3] // the C library reads three
|
||||
}
|
||||
bound := waitOf(string(raw))
|
||||
type question struct {
|
||||
name string
|
||||
qtype uint16
|
||||
mesh bool
|
||||
}
|
||||
var questions []question
|
||||
if j.MeshName != "" {
|
||||
questions = append(questions, question{j.MeshName, TypeA, true}, question{j.MeshName, TypeAAAA, true})
|
||||
}
|
||||
questions = append(questions, question{PublicName, TypeA, false})
|
||||
|
||||
// Every question at once, so a look takes one bound however many resolvers are silent.
|
||||
type result struct {
|
||||
answer Answer
|
||||
err error
|
||||
}
|
||||
results := make([][]result, len(servers))
|
||||
var wg sync.WaitGroup
|
||||
for si, server := range servers {
|
||||
results[si] = make([]result, len(questions))
|
||||
for qi, q := range questions {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a, err := j.m.Ask(ctx, server, q.name, q.qtype, bound)
|
||||
results[si][qi] = result{a, err}
|
||||
}()
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var failing, said []string
|
||||
meshFails, publicFails := 0, 0
|
||||
for si, server := range servers {
|
||||
var wrong []string
|
||||
for qi, q := range questions {
|
||||
a, err := results[si][qi].answer, results[si][qi].err
|
||||
word := ""
|
||||
switch {
|
||||
case err != nil:
|
||||
word = err.Error()
|
||||
case q.qtype == TypeAAAA:
|
||||
// The mesh's names carry no IPv6 address: "none", never "no such name" (issue 262).
|
||||
if a.Rcode != RcodeOK {
|
||||
word = "says " + rcodeWords(a.Rcode) + " for its IPv6 address, where it should say there is none"
|
||||
}
|
||||
case a.Rcode != RcodeOK:
|
||||
word = "says " + rcodeWords(a.Rcode)
|
||||
case a.Records == 0:
|
||||
word = "answers no address"
|
||||
}
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
wrong = append(wrong, fmt.Sprintf("%s %s: %s", q.name, typeWords(q.qtype), word))
|
||||
if q.mesh {
|
||||
meshFails++
|
||||
} else {
|
||||
publicFails++
|
||||
}
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
failing = append(failing, server)
|
||||
said = append(said, server+" — "+strings.Join(wrong, "; "))
|
||||
}
|
||||
}
|
||||
if len(failing) == 0 {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
var reason string
|
||||
switch {
|
||||
case len(failing) < len(servers):
|
||||
reason = fmt.Sprintf("%d of its %d resolvers do not answer as the mesh's do", len(failing), len(servers))
|
||||
case meshFails > 0 && publicFails > 0:
|
||||
reason = "neither mesh names nor public names resolve"
|
||||
case meshFails > 0:
|
||||
reason = "mesh names do not resolve"
|
||||
default:
|
||||
reason = "public names do not resolve"
|
||||
}
|
||||
return Finding{Reason: reason, Said: fmt.Sprintf("within %s: %s", bound, strings.Join(said, " | ")), Toward: failing}
|
||||
}
|
||||
|
||||
// lookTunnel reads the mesh's interface: on a machine reaching the hub, the hub's handshake; on the hub,
|
||||
// whether any machine has handshaken with it.
|
||||
func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
||||
if j.m.Run == nil {
|
||||
return Finding{Skip: true}
|
||||
}
|
||||
hs, err := j.m.Run(ctx, "wg", "show", Interface, "latest-handshakes")
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "executable file not found") {
|
||||
return Finding{Skip: true}
|
||||
}
|
||||
return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()),
|
||||
Toward: []string{TowardHub}}
|
||||
}
|
||||
ips, err := j.m.Run(ctx, "wg", "show", Interface, "allowed-ips")
|
||||
if err != nil {
|
||||
return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()),
|
||||
Toward: []string{TowardHub}}
|
||||
}
|
||||
handshakes := map[string]int64{}
|
||||
for _, line := range strings.Split(hs, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 2 {
|
||||
at, _ := strconv.ParseInt(f[1], 10, 64)
|
||||
handshakes[f[0]] = at
|
||||
}
|
||||
}
|
||||
hub := ""
|
||||
for _, line := range strings.Split(ips, "\n") {
|
||||
f := strings.Fields(line)
|
||||
for _, prefix := range f[min(1, len(f)):] {
|
||||
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
|
||||
hub = f[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
age := func(at int64) string {
|
||||
if at == 0 {
|
||||
return "never"
|
||||
}
|
||||
return now.Sub(time.Unix(at, 0)).Round(time.Second).String() + " ago"
|
||||
}
|
||||
if hub != "" {
|
||||
at := handshakes[hub]
|
||||
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
|
||||
}
|
||||
if len(handshakes) == 0 {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
var newest int64
|
||||
for _, at := range handshakes {
|
||||
newest = max(newest, at)
|
||||
}
|
||||
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
|
||||
}
|
||||
|
||||
func (j *Judge) lookBus() Finding {
|
||||
if j.m.Linked == nil {
|
||||
return Finding{Skip: true}
|
||||
}
|
||||
if j.m.Linked() {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
return Finding{Reason: "the bus cannot be reached", Said: "no link to the bus is open", Toward: []string{TowardHub}}
|
||||
}
|
||||
|
||||
func (j *Judge) lookRoute() Finding {
|
||||
var routes []string
|
||||
for _, table := range []struct {
|
||||
file string
|
||||
dest, mask, i int
|
||||
}{{"route", 1, 7, 0}, {"ipv6_route", 0, 1, 9}} {
|
||||
f, err := os.Open(filepath.Join(j.m.ProcNet, table.file))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) <= max(table.dest, table.mask, table.i) || fields[0] == "Iface" {
|
||||
continue
|
||||
}
|
||||
if zero(fields[table.dest]) && zero(fields[table.mask]) && fields[table.i] != "lo" {
|
||||
routes = append(routes, fields[table.i])
|
||||
}
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
if len(routes) > 0 {
|
||||
return Finding{OK: true}
|
||||
}
|
||||
return Finding{Reason: "the machine has no default route", Said: "no default route in " + j.m.ProcNet}
|
||||
}
|
||||
|
||||
// nameservers is every resolver a file lists, in order.
|
||||
func nameservers(content string) []string {
|
||||
var out []string
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 2 && f[0] == "nameserver" {
|
||||
out = append(out, f[1])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitOf is how long the file tells the C library to wait for one resolver: `options timeout:n`, five
|
||||
// seconds when it says nothing, and never under one.
|
||||
func waitOf(content string) time.Duration {
|
||||
wait := 5 * time.Second
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 0 || f[0] != "options" {
|
||||
continue
|
||||
}
|
||||
for _, o := range f[1:] {
|
||||
if v, ok := strings.CutPrefix(o, "timeout:"); ok {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
wait = time.Duration(max(n, 1)) * time.Second
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return wait
|
||||
}
|
||||
|
||||
func rcodeWords(rcode int) string {
|
||||
switch rcode {
|
||||
case RcodeNXDomain:
|
||||
return "no such name"
|
||||
case RcodeServFail:
|
||||
return "it failed"
|
||||
case RcodeRefused:
|
||||
return "it refuses"
|
||||
}
|
||||
return fmt.Sprintf("code %d", rcode)
|
||||
}
|
||||
|
||||
func typeWords(t uint16) string {
|
||||
if t == TypeAAAA {
|
||||
return "(IPv6)"
|
||||
}
|
||||
return "(IPv4)"
|
||||
}
|
||||
|
||||
func listOrNone(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "no resolver"
|
||||
}
|
||||
return strings.Join(s, ", ")
|
||||
}
|
||||
|
||||
func zero(hex string) bool { return strings.Trim(hex, "0") == "" }
|
||||
|
||||
func firstLine(s string) string {
|
||||
line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
|
||||
return line
|
||||
}
|
||||
|
||||
// running is the names of the programs running, from /proc.
|
||||
func running() []string {
|
||||
entries, err := os.ReadDir("/proc")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if _, err := strconv.Atoi(e.Name()); err != nil {
|
||||
continue
|
||||
}
|
||||
comm, err := os.ReadFile(filepath.Join("/proc", e.Name(), "comm"))
|
||||
if err == nil {
|
||||
seen[strings.TrimSpace(string(comm))] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for n := range seen {
|
||||
out = append(out, n)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
package network
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0241, "how it is checked": the file rewritten by another program is said on the second
|
||||
// look, naming the writer; written back, healthy on the first; a resolver answering NXDOMAIN for a mesh
|
||||
// name's IPv6 address is a finding (issue 262); a stale handshake with the hub, the bus unlinked and no
|
||||
// default route are each said; one failing look is not a finding.
|
||||
|
||||
const meshFile = `# Managed by the mesh, and written by the module holding this machine's uplink.
|
||||
nameserver 10.10.0.2
|
||||
nameserver 10.10.0.1
|
||||
options timeout:1 attempts:2 edns0
|
||||
`
|
||||
|
||||
// vpnFile is what the VPN client writes on connect, its header as it writes it.
|
||||
const vpnFile = `# Dynamic resolv.conf(5) file for glibc resolver(3) generated by forticlient
|
||||
# The original file is backed up and will be restored after the VPN disconnects.
|
||||
nameserver 172.16.5.5
|
||||
nameserver 172.16.5.6
|
||||
search corp.example
|
||||
`
|
||||
|
||||
type fakeMachine struct {
|
||||
dir string
|
||||
now time.Time
|
||||
linked bool
|
||||
answers map[string]Answer // server|name|type
|
||||
wrong map[string]error
|
||||
hs, ips string
|
||||
wgErr error
|
||||
running []string
|
||||
}
|
||||
|
||||
func newFake(t *testing.T) *fakeMachine {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
f := &fakeMachine{dir: dir, now: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC), linked: true,
|
||||
answers: map[string]Answer{}, wrong: map[string]error{}}
|
||||
f.write(t, meshFile)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "net"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.route(t, true)
|
||||
f.hub(f.now.Add(-time.Minute))
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeMachine) write(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(f.dir, "resolv.conf")
|
||||
os.Remove(path)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMachine) route(t *testing.T, has bool) {
|
||||
t.Helper()
|
||||
table := "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\n" +
|
||||
"mesh0\t00000A0A\t00000000\t0001\t0\t0\t0\t00FFFFFF\n"
|
||||
if has {
|
||||
table += "wlan0\t00000000\t0101A8C0\t0003\t0\t0\t600\t00000000\n"
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(f.dir, "net", "route"), []byte(table), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// hub is a machine reaching the hub, its newest handshake at at.
|
||||
func (f *fakeMachine) hub(at time.Time) {
|
||||
f.hs = "HUBKEY=\t" + itoa(at.Unix()) + "\n"
|
||||
f.ips = "HUBKEY=\t10.10.0.0/24\n"
|
||||
}
|
||||
|
||||
func itoa(n int64) string { return strconv.FormatInt(n, 10) }
|
||||
|
||||
func (f *fakeMachine) judge(t *testing.T) *Judge {
|
||||
t.Helper()
|
||||
j := New(Machine{
|
||||
ResolvPath: filepath.Join(f.dir, "resolv.conf"),
|
||||
ProcNet: filepath.Join(f.dir, "net"),
|
||||
Ask: func(_ context.Context, server, name string, qtype uint16, _ time.Duration) (Answer, error) {
|
||||
key := server + "|" + name + "|" + typeWords(qtype)
|
||||
if err, ok := f.wrong[key]; ok {
|
||||
return Answer{}, err
|
||||
}
|
||||
if a, ok := f.answers[key]; ok {
|
||||
return a, nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(server, "10.10.") && qtype == TypeAAAA:
|
||||
return Answer{}, nil // the mesh's names have no IPv6 address: none, not no such name
|
||||
case strings.HasPrefix(server, "10.10."):
|
||||
return Answer{Records: 1}, nil
|
||||
case name == "novox.internal":
|
||||
return Answer{Rcode: RcodeNXDomain}, nil // the VPN's resolver knows no mesh name
|
||||
}
|
||||
return Answer{Records: 1}, nil
|
||||
},
|
||||
Run: func(_ context.Context, name string, args ...string) (string, error) {
|
||||
if f.wgErr != nil {
|
||||
return "", f.wgErr
|
||||
}
|
||||
if args[len(args)-1] == "latest-handshakes" {
|
||||
return f.hs, nil
|
||||
}
|
||||
return f.ips, nil
|
||||
},
|
||||
Linked: func() bool { return f.linked },
|
||||
Running: func() []string { return f.running },
|
||||
Now: func() time.Time { return f.now },
|
||||
}, "novox.internal")
|
||||
j.Declare(meshFile, "networkmanager", true)
|
||||
return j
|
||||
}
|
||||
|
||||
// look moves the clock a look on and looks.
|
||||
func (f *fakeMachine) look(t *testing.T, j *Judge) Statement {
|
||||
t.Helper()
|
||||
f.now = f.now.Add(LookEvery)
|
||||
st, _ := j.Look(t.Context())
|
||||
return st
|
||||
}
|
||||
|
||||
func partOf(st Statement, name string) (Part, bool) {
|
||||
for _, p := range st.Parts {
|
||||
if p.Part == name {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return Part{}, false
|
||||
}
|
||||
|
||||
func TestAHealthyMachineIsSaidHealthyOnItsFirstLook(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
st := f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("a healthy machine is said %s: %+v", st.State, st.Parts)
|
||||
}
|
||||
if len(st.Parts) != len(Parts) {
|
||||
t.Fatalf("want every part judged, got %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFileRewrittenByAVPNClientIsSaidOnTheSecondLookNamingItAndClearedWhenWrittenBack(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.look(t, j)
|
||||
f.write(t, vpnFile)
|
||||
|
||||
st := f.look(t, j)
|
||||
if st.State == Unhealthy {
|
||||
t.Fatalf("one look raised it: %+v", st.Parts)
|
||||
}
|
||||
if p, _ := partOf(st, PartResolvConf); p.State != Healthy || p.Streak != 1 {
|
||||
t.Fatalf("after one failing look the file is %+v; want still healthy, a streak of one", p)
|
||||
}
|
||||
|
||||
st = f.look(t, j)
|
||||
if st.State != Unhealthy {
|
||||
t.Fatalf("two looks did not make it unhealthy: %+v", st.Parts)
|
||||
}
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.State != Unhealthy || p.Writer != "FortiClient" || p.Owner != "networkmanager" {
|
||||
t.Fatalf("the file is said %+v; want unhealthy, written by FortiClient, owned by networkmanager", p)
|
||||
}
|
||||
if strings.Contains(p.Reason, "172.16.") || !strings.Contains(p.Said, "172.16.5.5") {
|
||||
t.Fatalf("the addresses belong in what is said, never the reason: %+v", p)
|
||||
}
|
||||
// And the mesh's names do not resolve through what the VPN client wrote.
|
||||
names, _ := partOf(st, PartNames)
|
||||
if names.State != Unhealthy || names.Reason != "mesh names do not resolve" {
|
||||
t.Fatalf("names through the VPN's resolvers are said %+v", names)
|
||||
}
|
||||
|
||||
f.write(t, meshFile)
|
||||
st = f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("written back, it is still %s: %+v", st.State, st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWriterIsNamedByWhatRunsWhenTheFileSaysNothing(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.running = []string{"systemd", "openvpn"}
|
||||
j := f.judge(t)
|
||||
f.write(t, "nameserver 192.0.2.53\n")
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.Writer != "OpenVPN?" || !strings.Contains(p.Said, "openvpn is running") {
|
||||
t.Fatalf("want the running VPN client named as a guess, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALinkInPlaceOfTheFileNamesWhatItPointsAt(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
target := filepath.Join(f.dir, "stub-resolv.conf")
|
||||
if err := os.WriteFile(target, []byte(meshFile), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(f.dir, "systemd", "resolve")
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(target, filepath.Join(path, "stub-resolv.conf")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Remove(filepath.Join(f.dir, "resolv.conf"))
|
||||
if err := os.Symlink(filepath.Join(path, "stub-resolv.conf"), filepath.Join(f.dir, "resolv.conf")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.State != Unhealthy || p.Writer != "systemd-resolved" {
|
||||
t.Fatalf("a link is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingDeclaredIsNotJudged(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
j.Declare("", "", false)
|
||||
f.write(t, vpnFile)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
if _, judged := partOf(st, PartResolvConf); judged {
|
||||
t.Fatalf("a file nothing declares was judged: %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNXDomainForAMeshNamesIPv6AddressIsAFinding(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.answers["10.10.0.1|novox.internal|(IPv6)"] = Answer{Rcode: RcodeNXDomain}
|
||||
j := f.judge(t)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartNames)
|
||||
if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.1" ||
|
||||
p.Reason != "1 of its 2 resolvers do not answer as the mesh's do" || !strings.Contains(p.Said, "IPv6") {
|
||||
t.Fatalf("issue 262's answer is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResolverThatDoesNotAnswerIsNamedAndOneLookIsNotAFinding(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s")
|
||||
st := f.look(t, j)
|
||||
if p, _ := partOf(st, PartNames); p.State == Unhealthy {
|
||||
t.Fatalf("one unanswered question raised it: %+v", p)
|
||||
}
|
||||
delete(f.wrong, "10.10.0.2|novox.internal|(IPv4)")
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
t.Fatalf("answered again, it is %s", st.State)
|
||||
}
|
||||
f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s")
|
||||
f.look(t, j)
|
||||
st = f.look(t, j)
|
||||
p, _ := partOf(st, PartNames)
|
||||
if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.2" {
|
||||
t.Fatalf("a silent resolver is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTunnelTheBusAndTheRouteAreEachSaid(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.hub(f.now.Add(-10 * time.Minute))
|
||||
f.linked = false
|
||||
f.route(t, false)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
for _, name := range []string{PartTunnel, PartBus, PartRoute} {
|
||||
p, _ := partOf(st, name)
|
||||
if p.State != Unhealthy {
|
||||
t.Fatalf("%s is said %+v", name, p)
|
||||
}
|
||||
}
|
||||
if p, _ := partOf(st, PartTunnel); len(p.Toward) != 1 || p.Toward[0] != TowardHub {
|
||||
t.Fatalf("the tunnel's failure does not point at the hub: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
||||
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
||||
j := f.judge(t)
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.wgErr = errors.New(`exec: "wg": executable file not found in $PATH`)
|
||||
j := f.judge(t)
|
||||
st := f.look(t, j)
|
||||
if _, judged := partOf(st, PartTunnel); judged {
|
||||
t.Fatal("a machine without wg judged a tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBetweenLooksTheLastStatementIsAnswered(t *testing.T) {
|
||||
f := newFake(t)
|
||||
var calls atomic.Int64
|
||||
j := f.judge(t)
|
||||
ask := j.m.Ask
|
||||
j.m.Ask = func(ctx context.Context, s, n string, q uint16, d time.Duration) (Answer, error) {
|
||||
calls.Add(1)
|
||||
return ask(ctx, s, n, q, d)
|
||||
}
|
||||
f.look(t, j)
|
||||
before := calls.Load()
|
||||
f.now = f.now.Add(LookEvery / 2)
|
||||
if _, changed := j.Look(t.Context()); changed || calls.Load() != before {
|
||||
t.Fatalf("a look half a period later asked again (%d questions) or said a change", calls.Load()-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWaitIsTheFilesOwn(t *testing.T) {
|
||||
if w := waitOf(meshFile); w != time.Second {
|
||||
t.Fatalf("timeout:1 is %s", w)
|
||||
}
|
||||
if w := waitOf("nameserver 192.0.2.1\n"); w != 5*time.Second {
|
||||
t.Fatalf("no options is %s", w)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAskReadsARealAnswer asks a resolver this test raises: an address for one name, none for its IPv6
|
||||
// address, and no such name for another.
|
||||
func TestAskReadsARealAnswer(t *testing.T) {
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Skip("no UDP here:", err)
|
||||
}
|
||||
defer pc.Close()
|
||||
go func() {
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
n, from, err := pc.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
q := append([]byte(nil), buf[:n]...)
|
||||
resp := append([]byte(nil), q...)
|
||||
resp[2] |= 0x80
|
||||
qtype := uint16(q[n-4])<<8 | uint16(q[n-3])
|
||||
switch {
|
||||
case strings.Contains(string(q), "missing"):
|
||||
resp[3] = RcodeNXDomain
|
||||
case qtype == TypeA:
|
||||
resp[7] = 1
|
||||
resp = append(resp, 0xc0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1)
|
||||
}
|
||||
pc.WriteTo(resp, from)
|
||||
}
|
||||
}()
|
||||
server := pc.LocalAddr().String()
|
||||
ctx := t.Context()
|
||||
if a, err := Ask(ctx, server, "novox.internal", TypeA, time.Second); err != nil || a.Rcode != 0 || a.Records != 1 {
|
||||
t.Fatalf("A: %+v %v", a, err)
|
||||
}
|
||||
if a, err := Ask(ctx, server, "novox.internal", TypeAAAA, time.Second); err != nil || a.Rcode != 0 || a.Records != 0 {
|
||||
t.Fatalf("AAAA: %+v %v", a, err)
|
||||
}
|
||||
if a, err := Ask(ctx, server, "missing.internal", TypeA, time.Second); err != nil || a.Rcode != RcodeNXDomain {
|
||||
t.Fatalf("NXDOMAIN: %+v %v", a, err)
|
||||
}
|
||||
if _, err := Ask(ctx, "127.0.0.1:9", "novox.internal", TypeA, 200*time.Millisecond); err == nil {
|
||||
t.Fatal("a resolver that does not answer answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupNamedForItsWriterNamesItWhateverTheFileSays(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
// The client renames the mesh's file aside: the backup keeps the old file's time.
|
||||
if err := os.WriteFile(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), []byte(meshFile), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := time.Now().Add(-time.Hour)
|
||||
os.Chtimes(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), old, old)
|
||||
f.write(t, "nameserver 192.0.2.53\n")
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
if p, _ := partOf(st, PartResolvConf); p.Writer != "FortiClient" || !strings.Contains(p.Said, "forticlient.backup") {
|
||||
t.Fatalf("the backup did not name its writer: %+v", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package network
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Naming the program that rewrote the resolver file (ADR 0241 rule 2). **A guess, said as one**: the
|
||||
// mesh cannot see who wrote a file after the fact, only what the file, its link and the machine show. In
|
||||
// order of how much each says:
|
||||
//
|
||||
// 1. what the file says of itself — every program that writes it puts its name in a comment;
|
||||
// 2. a backup the writer left beside it — named for the writer, or changed when the file was;
|
||||
// 3. a program known to write the file, running now.
|
||||
//
|
||||
// Nothing found is said as nothing found, never as a name.
|
||||
|
||||
// signs are the words a writer leaves in the file's comments, and its name.
|
||||
var signs = []struct{ word, name string }{
|
||||
{"forti", "FortiClient"},
|
||||
{"openfortivpn", "openfortivpn"},
|
||||
{"networkmanager", "NetworkManager"},
|
||||
{"systemd-resolved", "systemd-resolved"},
|
||||
{"resolvconf", "resolvconf"},
|
||||
{"dhcpcd", "dhcpcd"},
|
||||
{"dhclient", "dhclient"},
|
||||
{"netconfig", "netconfig"},
|
||||
{"openvpn", "OpenVPN"},
|
||||
{"openconnect", "OpenConnect"},
|
||||
{"vpnc", "vpnc"},
|
||||
{"tailscale", "Tailscale"},
|
||||
{"connman", "ConnMan"},
|
||||
}
|
||||
|
||||
// writers are the programs known to rewrite the file, as they run, and their name. The VPN clients
|
||||
// first: they are what rewrites a file the machine's network manager was already told to keep off.
|
||||
var writers = []struct{ comm, name string }{
|
||||
{"fortivpn", "FortiClient"},
|
||||
{"forticlient", "FortiClient"},
|
||||
{"fctsched", "FortiClient"},
|
||||
{"openfortivpn", "openfortivpn"},
|
||||
{"openvpn", "OpenVPN"},
|
||||
{"openconnect", "OpenConnect"},
|
||||
{"vpnc", "vpnc"},
|
||||
{"charon", "strongSwan"},
|
||||
{"tailscaled", "Tailscale"},
|
||||
{"dhclient", "dhclient"},
|
||||
{"resolvconf", "resolvconf"},
|
||||
}
|
||||
|
||||
// writerOf names who rewrote the file, and why that name, from the file's own words, a backup changed
|
||||
// beside it, or a writer running.
|
||||
func (j *Judge) writerOf(content string, changed time.Time) (string, string) {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "#") && !strings.HasPrefix(line, ";") {
|
||||
continue
|
||||
}
|
||||
lower := strings.ToLower(line)
|
||||
for _, s := range signs {
|
||||
if strings.Contains(lower, s.word) {
|
||||
return s.name, "its own header names " + s.name
|
||||
}
|
||||
}
|
||||
}
|
||||
// A backup the writer kept beside it.
|
||||
backup := ""
|
||||
matches, _ := filepath.Glob(j.m.ResolvPath + "*")
|
||||
for _, m := range matches {
|
||||
if m == j.m.ResolvPath {
|
||||
continue
|
||||
}
|
||||
info, err := os.Stat(m)
|
||||
if err != nil || info.IsDir() {
|
||||
continue
|
||||
}
|
||||
// A backup that names its writer says so whenever it was made: a client that renames the file
|
||||
// aside (FortiClient does, on connect) leaves the backup with the old file's time, not its own.
|
||||
lower := strings.ToLower(filepath.Base(m))
|
||||
for _, s := range signs {
|
||||
if strings.Contains(lower, s.word) {
|
||||
return s.name, "it left " + m + " beside it"
|
||||
}
|
||||
}
|
||||
if d := info.ModTime().Sub(changed); d >= -time.Minute && d <= time.Minute {
|
||||
backup = m
|
||||
}
|
||||
}
|
||||
why := "no program it could be is known"
|
||||
if backup != "" {
|
||||
why = backup + " was changed when it was: whoever wrote it kept a copy there"
|
||||
}
|
||||
runningNow := map[string]bool{}
|
||||
for _, name := range j.m.Running() {
|
||||
runningNow[strings.ToLower(name)] = true
|
||||
}
|
||||
for _, w := range writers {
|
||||
if runningNow[w.comm] {
|
||||
return w.name + "?", w.comm + " is running; " + why
|
||||
}
|
||||
}
|
||||
return "", why
|
||||
}
|
||||
|
||||
// writerOfLink names the program a link points the file at.
|
||||
func writerOfLink(target string) string {
|
||||
lower := strings.ToLower(target)
|
||||
switch {
|
||||
case strings.Contains(lower, "systemd/resolve"):
|
||||
return "systemd-resolved"
|
||||
case strings.Contains(lower, "resolvconf"):
|
||||
return "resolvconf"
|
||||
case strings.Contains(lower, "networkmanager"):
|
||||
return "NetworkManager"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user