Judge the machine's own networking beside what its modules run (hq ADR 0241)
A VPN client rewrote the laptop's resolver file and every mesh name failed while each module read healthy: nothing asked the machine. The engine now looks every 30 s at the resolver file the uplink holder declared (naming the program that rewrote it), the names through each listed resolver (NXDOMAIN for a mesh name's IPv6 address is a finding, issue 262), the tunnel's handshake with the hub, the bus and the default route; a part is unhealthy on its second failing look, and the statement carries it.
This commit is contained in:
@@ -0,0 +1,406 @@
|
||||
package network
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0241, "how it is checked": the file rewritten by another program is said on the second
|
||||
// look, naming the writer; written back, healthy on the first; a resolver answering NXDOMAIN for a mesh
|
||||
// name's IPv6 address is a finding (issue 262); a stale handshake with the hub, the bus unlinked and no
|
||||
// default route are each said; one failing look is not a finding.
|
||||
|
||||
const meshFile = `# Managed by the mesh, and written by the module holding this machine's uplink.
|
||||
nameserver 10.10.0.2
|
||||
nameserver 10.10.0.1
|
||||
options timeout:1 attempts:2 edns0
|
||||
`
|
||||
|
||||
// vpnFile is what the VPN client writes on connect, its header as it writes it.
|
||||
const vpnFile = `# Dynamic resolv.conf(5) file for glibc resolver(3) generated by forticlient
|
||||
# The original file is backed up and will be restored after the VPN disconnects.
|
||||
nameserver 172.16.5.5
|
||||
nameserver 172.16.5.6
|
||||
search corp.example
|
||||
`
|
||||
|
||||
type fakeMachine struct {
|
||||
dir string
|
||||
now time.Time
|
||||
linked bool
|
||||
answers map[string]Answer // server|name|type
|
||||
wrong map[string]error
|
||||
hs, ips string
|
||||
wgErr error
|
||||
running []string
|
||||
}
|
||||
|
||||
func newFake(t *testing.T) *fakeMachine {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
f := &fakeMachine{dir: dir, now: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC), linked: true,
|
||||
answers: map[string]Answer{}, wrong: map[string]error{}}
|
||||
f.write(t, meshFile)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "net"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.route(t, true)
|
||||
f.hub(f.now.Add(-time.Minute))
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fakeMachine) write(t *testing.T, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(f.dir, "resolv.conf")
|
||||
os.Remove(path)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMachine) route(t *testing.T, has bool) {
|
||||
t.Helper()
|
||||
table := "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\n" +
|
||||
"mesh0\t00000A0A\t00000000\t0001\t0\t0\t0\t00FFFFFF\n"
|
||||
if has {
|
||||
table += "wlan0\t00000000\t0101A8C0\t0003\t0\t0\t600\t00000000\n"
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(f.dir, "net", "route"), []byte(table), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// hub is a machine reaching the hub, its newest handshake at at.
|
||||
func (f *fakeMachine) hub(at time.Time) {
|
||||
f.hs = "HUBKEY=\t" + itoa(at.Unix()) + "\n"
|
||||
f.ips = "HUBKEY=\t10.10.0.0/24\n"
|
||||
}
|
||||
|
||||
func itoa(n int64) string { return strconv.FormatInt(n, 10) }
|
||||
|
||||
func (f *fakeMachine) judge(t *testing.T) *Judge {
|
||||
t.Helper()
|
||||
j := New(Machine{
|
||||
ResolvPath: filepath.Join(f.dir, "resolv.conf"),
|
||||
ProcNet: filepath.Join(f.dir, "net"),
|
||||
Ask: func(_ context.Context, server, name string, qtype uint16, _ time.Duration) (Answer, error) {
|
||||
key := server + "|" + name + "|" + typeWords(qtype)
|
||||
if err, ok := f.wrong[key]; ok {
|
||||
return Answer{}, err
|
||||
}
|
||||
if a, ok := f.answers[key]; ok {
|
||||
return a, nil
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(server, "10.10.") && qtype == TypeAAAA:
|
||||
return Answer{}, nil // the mesh's names have no IPv6 address: none, not no such name
|
||||
case strings.HasPrefix(server, "10.10."):
|
||||
return Answer{Records: 1}, nil
|
||||
case name == "novox.internal":
|
||||
return Answer{Rcode: RcodeNXDomain}, nil // the VPN's resolver knows no mesh name
|
||||
}
|
||||
return Answer{Records: 1}, nil
|
||||
},
|
||||
Run: func(_ context.Context, name string, args ...string) (string, error) {
|
||||
if f.wgErr != nil {
|
||||
return "", f.wgErr
|
||||
}
|
||||
if args[len(args)-1] == "latest-handshakes" {
|
||||
return f.hs, nil
|
||||
}
|
||||
return f.ips, nil
|
||||
},
|
||||
Linked: func() bool { return f.linked },
|
||||
Running: func() []string { return f.running },
|
||||
Now: func() time.Time { return f.now },
|
||||
}, "novox.internal")
|
||||
j.Declare(meshFile, "networkmanager", true)
|
||||
return j
|
||||
}
|
||||
|
||||
// look moves the clock a look on and looks.
|
||||
func (f *fakeMachine) look(t *testing.T, j *Judge) Statement {
|
||||
t.Helper()
|
||||
f.now = f.now.Add(LookEvery)
|
||||
st, _ := j.Look(t.Context())
|
||||
return st
|
||||
}
|
||||
|
||||
func partOf(st Statement, name string) (Part, bool) {
|
||||
for _, p := range st.Parts {
|
||||
if p.Part == name {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return Part{}, false
|
||||
}
|
||||
|
||||
func TestAHealthyMachineIsSaidHealthyOnItsFirstLook(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
st := f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("a healthy machine is said %s: %+v", st.State, st.Parts)
|
||||
}
|
||||
if len(st.Parts) != len(Parts) {
|
||||
t.Fatalf("want every part judged, got %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFileRewrittenByAVPNClientIsSaidOnTheSecondLookNamingItAndClearedWhenWrittenBack(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.look(t, j)
|
||||
f.write(t, vpnFile)
|
||||
|
||||
st := f.look(t, j)
|
||||
if st.State == Unhealthy {
|
||||
t.Fatalf("one look raised it: %+v", st.Parts)
|
||||
}
|
||||
if p, _ := partOf(st, PartResolvConf); p.State != Healthy || p.Streak != 1 {
|
||||
t.Fatalf("after one failing look the file is %+v; want still healthy, a streak of one", p)
|
||||
}
|
||||
|
||||
st = f.look(t, j)
|
||||
if st.State != Unhealthy {
|
||||
t.Fatalf("two looks did not make it unhealthy: %+v", st.Parts)
|
||||
}
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.State != Unhealthy || p.Writer != "FortiClient" || p.Owner != "networkmanager" {
|
||||
t.Fatalf("the file is said %+v; want unhealthy, written by FortiClient, owned by networkmanager", p)
|
||||
}
|
||||
if strings.Contains(p.Reason, "172.16.") || !strings.Contains(p.Said, "172.16.5.5") {
|
||||
t.Fatalf("the addresses belong in what is said, never the reason: %+v", p)
|
||||
}
|
||||
// And the mesh's names do not resolve through what the VPN client wrote.
|
||||
names, _ := partOf(st, PartNames)
|
||||
if names.State != Unhealthy || names.Reason != "mesh names do not resolve" {
|
||||
t.Fatalf("names through the VPN's resolvers are said %+v", names)
|
||||
}
|
||||
|
||||
f.write(t, meshFile)
|
||||
st = f.look(t, j)
|
||||
if st.State != Healthy {
|
||||
t.Fatalf("written back, it is still %s: %+v", st.State, st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWriterIsNamedByWhatRunsWhenTheFileSaysNothing(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.running = []string{"systemd", "openvpn"}
|
||||
j := f.judge(t)
|
||||
f.write(t, "nameserver 192.0.2.53\n")
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.Writer != "OpenVPN?" || !strings.Contains(p.Said, "openvpn is running") {
|
||||
t.Fatalf("want the running VPN client named as a guess, got %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALinkInPlaceOfTheFileNamesWhatItPointsAt(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
target := filepath.Join(f.dir, "stub-resolv.conf")
|
||||
if err := os.WriteFile(target, []byte(meshFile), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(f.dir, "systemd", "resolve")
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(target, filepath.Join(path, "stub-resolv.conf")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Remove(filepath.Join(f.dir, "resolv.conf"))
|
||||
if err := os.Symlink(filepath.Join(path, "stub-resolv.conf"), filepath.Join(f.dir, "resolv.conf")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartResolvConf)
|
||||
if p.State != Unhealthy || p.Writer != "systemd-resolved" {
|
||||
t.Fatalf("a link is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingDeclaredIsNotJudged(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
j.Declare("", "", false)
|
||||
f.write(t, vpnFile)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
if _, judged := partOf(st, PartResolvConf); judged {
|
||||
t.Fatalf("a file nothing declares was judged: %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNXDomainForAMeshNamesIPv6AddressIsAFinding(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.answers["10.10.0.1|novox.internal|(IPv6)"] = Answer{Rcode: RcodeNXDomain}
|
||||
j := f.judge(t)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
p, _ := partOf(st, PartNames)
|
||||
if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.1" ||
|
||||
p.Reason != "1 of its 2 resolvers do not answer as the mesh's do" || !strings.Contains(p.Said, "IPv6") {
|
||||
t.Fatalf("issue 262's answer is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResolverThatDoesNotAnswerIsNamedAndOneLookIsNotAFinding(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s")
|
||||
st := f.look(t, j)
|
||||
if p, _ := partOf(st, PartNames); p.State == Unhealthy {
|
||||
t.Fatalf("one unanswered question raised it: %+v", p)
|
||||
}
|
||||
delete(f.wrong, "10.10.0.2|novox.internal|(IPv4)")
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
t.Fatalf("answered again, it is %s", st.State)
|
||||
}
|
||||
f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s")
|
||||
f.look(t, j)
|
||||
st = f.look(t, j)
|
||||
p, _ := partOf(st, PartNames)
|
||||
if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.2" {
|
||||
t.Fatalf("a silent resolver is said %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTunnelTheBusAndTheRouteAreEachSaid(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
f.hub(f.now.Add(-10 * time.Minute))
|
||||
f.linked = false
|
||||
f.route(t, false)
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
for _, name := range []string{PartTunnel, PartBus, PartRoute} {
|
||||
p, _ := partOf(st, name)
|
||||
if p.State != Unhealthy {
|
||||
t.Fatalf("%s is said %+v", name, p)
|
||||
}
|
||||
}
|
||||
if p, _ := partOf(st, PartTunnel); len(p.Toward) != 1 || p.Toward[0] != TowardHub {
|
||||
t.Fatalf("the tunnel's failure does not point at the hub: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
||||
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
||||
j := f.judge(t)
|
||||
if st := f.look(t, j); st.State != Healthy {
|
||||
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.wgErr = errors.New(`exec: "wg": executable file not found in $PATH`)
|
||||
j := f.judge(t)
|
||||
st := f.look(t, j)
|
||||
if _, judged := partOf(st, PartTunnel); judged {
|
||||
t.Fatal("a machine without wg judged a tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBetweenLooksTheLastStatementIsAnswered(t *testing.T) {
|
||||
f := newFake(t)
|
||||
var calls atomic.Int64
|
||||
j := f.judge(t)
|
||||
ask := j.m.Ask
|
||||
j.m.Ask = func(ctx context.Context, s, n string, q uint16, d time.Duration) (Answer, error) {
|
||||
calls.Add(1)
|
||||
return ask(ctx, s, n, q, d)
|
||||
}
|
||||
f.look(t, j)
|
||||
before := calls.Load()
|
||||
f.now = f.now.Add(LookEvery / 2)
|
||||
if _, changed := j.Look(t.Context()); changed || calls.Load() != before {
|
||||
t.Fatalf("a look half a period later asked again (%d questions) or said a change", calls.Load()-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWaitIsTheFilesOwn(t *testing.T) {
|
||||
if w := waitOf(meshFile); w != time.Second {
|
||||
t.Fatalf("timeout:1 is %s", w)
|
||||
}
|
||||
if w := waitOf("nameserver 192.0.2.1\n"); w != 5*time.Second {
|
||||
t.Fatalf("no options is %s", w)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAskReadsARealAnswer asks a resolver this test raises: an address for one name, none for its IPv6
|
||||
// address, and no such name for another.
|
||||
func TestAskReadsARealAnswer(t *testing.T) {
|
||||
pc, err := net.ListenPacket("udp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Skip("no UDP here:", err)
|
||||
}
|
||||
defer pc.Close()
|
||||
go func() {
|
||||
buf := make([]byte, 512)
|
||||
for {
|
||||
n, from, err := pc.ReadFrom(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
q := append([]byte(nil), buf[:n]...)
|
||||
resp := append([]byte(nil), q...)
|
||||
resp[2] |= 0x80
|
||||
qtype := uint16(q[n-4])<<8 | uint16(q[n-3])
|
||||
switch {
|
||||
case strings.Contains(string(q), "missing"):
|
||||
resp[3] = RcodeNXDomain
|
||||
case qtype == TypeA:
|
||||
resp[7] = 1
|
||||
resp = append(resp, 0xc0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1)
|
||||
}
|
||||
pc.WriteTo(resp, from)
|
||||
}
|
||||
}()
|
||||
server := pc.LocalAddr().String()
|
||||
ctx := t.Context()
|
||||
if a, err := Ask(ctx, server, "novox.internal", TypeA, time.Second); err != nil || a.Rcode != 0 || a.Records != 1 {
|
||||
t.Fatalf("A: %+v %v", a, err)
|
||||
}
|
||||
if a, err := Ask(ctx, server, "novox.internal", TypeAAAA, time.Second); err != nil || a.Rcode != 0 || a.Records != 0 {
|
||||
t.Fatalf("AAAA: %+v %v", a, err)
|
||||
}
|
||||
if a, err := Ask(ctx, server, "missing.internal", TypeA, time.Second); err != nil || a.Rcode != RcodeNXDomain {
|
||||
t.Fatalf("NXDOMAIN: %+v %v", a, err)
|
||||
}
|
||||
if _, err := Ask(ctx, "127.0.0.1:9", "novox.internal", TypeA, 200*time.Millisecond); err == nil {
|
||||
t.Fatal("a resolver that does not answer answered")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupNamedForItsWriterNamesItWhateverTheFileSays(t *testing.T) {
|
||||
f := newFake(t)
|
||||
j := f.judge(t)
|
||||
// The client renames the mesh's file aside: the backup keeps the old file's time.
|
||||
if err := os.WriteFile(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), []byte(meshFile), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := time.Now().Add(-time.Hour)
|
||||
os.Chtimes(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), old, old)
|
||||
f.write(t, "nameserver 192.0.2.53\n")
|
||||
f.look(t, j)
|
||||
st := f.look(t, j)
|
||||
if p, _ := partOf(st, PartResolvConf); p.Writer != "FortiClient" || !strings.Contains(p.Said, "forticlient.backup") {
|
||||
t.Fatalf("the backup did not name its writer: %+v", p)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user