From 429ea42357df0291b2f68e43f5d8f65c8bcc68d3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 Oct 2026 18:43:39 +0200 Subject: [PATCH] Judge the machine's own networking beside what its modules run (hq ADR 0241) A VPN client rewrote the laptop's resolver file and every mesh name failed while each module read healthy: nothing asked the machine. The engine now looks every 30 s at the resolver file the uplink holder declared (naming the program that rewrote it), the names through each listed resolver (NXDOMAIN for a mesh name's IPv6 address is a finding, issue 262), the tunnel's handshake with the hub, the bus and the default route; a part is unhealthy on its second failing look, and the statement carries it. --- cmd/mesh-host/main.go | 109 +++++- cmd/mesh-host/network_test.go | 54 +++ internal/link/messages.go | 30 ++ internal/link/queue.go | 9 + internal/network/dns.go | 155 ++++++++ internal/network/network.go | 598 +++++++++++++++++++++++++++++++ internal/network/network_test.go | 406 +++++++++++++++++++++ internal/network/writer.go | 119 ++++++ 8 files changed, 1476 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-host/network_test.go create mode 100644 internal/network/dns.go create mode 100644 internal/network/network.go create mode 100644 internal/network/network_test.go create mode 100644 internal/network/writer.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index e716625..20e4cee 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -17,6 +17,7 @@ import ( "flag" "fmt" "io" + "net" "os" "os/signal" "path/filepath" @@ -35,6 +36,7 @@ import ( "github.com/novox/mesh-host/internal/inventory" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/network" "github.com/novox/mesh-host/internal/outward" "github.com/novox/mesh-host/internal/profile" "github.com/novox/mesh-host/internal/reachable" @@ -1168,6 +1170,10 @@ func runLink(ctx context.Context, opts options) error { // spaced to the budget (ADR 0240 Phase B); a tool is asked of this machine's node tools over the // link open at the time. judging.Probes = &liveness.Probes{AskTool: queue.AskTool} + // And the machine's own networking (novox/hq ADR 0241): the resolver file the uplink holder + // declared, the names through it, the tunnel, the bus and the route — said in the same statement. + netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked}, + hostOf(mine.Membership.Broker))) go judging.Probe(aside) go judgeWhatRuns(aside, judging, queue, say) } @@ -1361,6 +1367,74 @@ const ReconcileEvery = 5 * time.Minute // reports no health, and in a test. var judging *liveness.Judge +// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a +// one-shot command and in a test, which say nothing of the network. +var netJudge networkJudge + +type networkJudge struct { + mu sync.Mutex + m *network.Judge + // The resolver file the last apply declared, kept for a judge made after it. + content, owner string + declared bool +} + +func (n *networkJudge) get() *network.Judge { + n.mu.Lock() + defer n.mu.Unlock() + return n.m +} + +func (n *networkJudge) set(m *network.Judge) { + n.mu.Lock() + defer n.mu.Unlock() + n.m = m + m.Declare(n.content, n.owner, n.declared) +} + +// declare is the resolver file an apply just applied, for the judge now and any made later. +func (n *networkJudge) declare(content, owner string, ok bool) { + n.mu.Lock() + defer n.mu.Unlock() + n.content, n.owner, n.declared = content, owner, ok + if n.m != nil { + n.m.Declare(content, owner, ok) + } +} + +// hostOf is the bus's name without its port: the mesh name the machine needs most. Empty when the bus +// is reached by address, and then no mesh name is asked. +func hostOf(broker string) string { + host := broker + if h, _, err := net.SplitHostPort(broker); err == nil { + host = h + } + if net.ParseIP(host) != nil { + return "" + } + return host +} + +// declaredResolvConf is the resolver file a declaration has a module write whole — the uplink holder's +// (ADR 0223) — and that module. +func declaredResolvConf(d *declaration.Declaration) (string, string, bool) { + if d == nil { + return "", "", false + } + for _, r := range d.Resources { + f, ok := r.(*declaration.File) + if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" { + continue + } + module, ok := liveness.ModuleOf(f.ID) + if !ok { + return "", "", false + } + return f.Content, module, true + } + return "", "", false +} + // How often a statement of health is said between reports: again every minute while anything is not // healthy (to-be 48 §4), so a lost event is not a lost fault; and every five minutes anyway, so a // controller that restarted knows a healthy machine's state without waiting for its next apply. @@ -1387,6 +1461,19 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } st, changed := j.Look(ctx) owed = owed || changed + var netSt *network.Statement + if n := netJudge.get(); n != nil { + ns, netChanged := n.Look(ctx) + netSt = &ns + owed = owed || netChanged + if netChanged { + for _, p := range ns.Parts { + if p.State == network.Unhealthy { + say(fmt.Sprintf("this machine's network is unhealthy: %s: %s (%s)", p.Part, p.Reason, p.Said)) + } + } + } + } // Never more looks than the budget (ADR 0240): said when the engine has to space its own out. if sp := j.Spacing(); sp != spaced { if sp > 1 { @@ -1396,7 +1483,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa spaced = sp } since := time.Since(lastSaid) - if !owed && !(!st.Healthy() && since >= sayUnhealthyAgain) && since < sayAnyway { + healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) + if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } if changed { @@ -1407,14 +1495,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa } } } - if queue.SayHealth(ctx, *healthAsReported(st)) { + if queue.SayHealth(ctx, *healthAsReported(st, netSt)) { lastSaid, owed = time.Now(), false } } } // healthAsReported is a statement as the report and the event carry it. -func healthAsReported(st liveness.Statement) *link.Health { +func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { // ReadinessContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase // B), which is what tells the controller it may be sent the field. h := &link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} @@ -1423,6 +1511,13 @@ func healthAsReported(st liveness.Statement) *link.Health { Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf()}) } + if ns != nil && ns.State != "" { + h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}} + for _, p := range ns.Parts { + h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason, + Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: p.Streak}) + } + } return h } @@ -1657,7 +1752,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto if j := judging; j != nil { j.Set(liveness.LongRunning(declared, held)) st, _ := j.Look(ctx) - report.Health = healthAsReported(st) + netJudge.declare(declaredResolvConf(declared)) + var netSt *network.Statement + if n := netJudge.get(); n != nil { + ns := n.Last() + netSt = &ns + } + report.Health = healthAsReported(st, netSt) } // Which of this machine's links face outside, for the filter the mesh writes around them // (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it, diff --git a/cmd/mesh-host/network_test.go b/cmd/mesh-host/network_test.go new file mode 100644 index 0000000..670702e --- /dev/null +++ b/cmd/mesh-host/network_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/network" +) + +// The machine's networking in the engine's statement (novox/hq ADR 0241): the file judged is the one the +// uplink holder declares whole, the mesh name asked is the bus's, and the statement carries the parts. + +func TestTheResolverFileJudgedIsTheOneAModuleDeclaresWhole(t *testing.T) { + d := &declaration.Declaration{Resources: []declaration.Resource{ + &declaration.File{ID: "hosts.file", Type: "file", Path: "/etc/hosts", Content: "x"}, + &declaration.File{ID: "networkmanager.resolv", Type: "file", Path: network.ResolvConf, Content: "nameserver 10.10.0.1\n"}, + }} + content, owner, ok := declaredResolvConf(d) + if !ok || owner != "networkmanager" || content != "nameserver 10.10.0.1\n" { + t.Fatalf("got %q %q %v", content, owner, ok) + } + d.Resources[1].(*declaration.File).CreateOnce = true + if _, _, ok := declaredResolvConf(d); ok { + t.Fatal("a seed nobody holds the machine to was judged as the declared file") + } + if _, _, ok := declaredResolvConf(nil); ok { + t.Fatal("no declaration declared a file") + } +} + +func TestTheMeshNameAskedIsTheBuses(t *testing.T) { + for broker, want := range map[string]string{"anchor.internal:4222": "anchor.internal", "192.0.2.10:5671": "", + "anchor.internal": "anchor.internal", "[2001:db8::1]:4222": ""} { + if got := hostOf(broker); got != want { + t.Errorf("%s: got %q, want %q", broker, got, want) + } + } +} + +func TestTheStatementCarriesTheNetwork(t *testing.T) { + at := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + h := healthAsReported(liveness.Statement{At: at}, &network.Statement{State: network.Unhealthy, Since: at, + Parts: []network.Part{{Part: network.PartResolvConf, State: network.Unhealthy, Writer: "FortiClient", + Owner: "networkmanager", Reason: "the resolver file was rewritten by another program", Since: at}}}) + if h.Network == nil || h.Network.State != network.Unhealthy || len(h.Network.Parts) != 1 || + h.Network.Parts[0].Writer != "FortiClient" || h.Network.Parts[0].Owner != "networkmanager" { + t.Fatalf("the statement says %+v", h.Network) + } + if h := healthAsReported(liveness.Statement{At: at}, nil); h.Network != nil { + t.Fatal("an engine with no network judge said a network") + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index c37263d..783d8d2 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -238,6 +238,36 @@ type Health struct { At time.Time `json:"at"` // Resources are every long-running resource of a module this machine runs, by module and id. Resources []ResourceHealth `json:"resources"` + // Network is the machine's own networking, judged by its engine (novox/hq ADR 0241): its resolver + // file, its names, its tunnel, its bus and its route. Absent from an engine older than that judging, + // which the controller reads as "not known", never as healthy. + Network *NetworkHealth `json:"network,omitempty"` +} + +// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since +// when, and each part. +type NetworkHealth struct { + State string `json:"state"` + Since time.Time `json:"since"` + Parts []NetworkPart `json:"parts"` +} + +// NetworkPart is one part of a machine's networking, as its engine judged it on its second look. +type NetworkPart struct { + // Part is resolv-conf, names, tunnel, bus or route. + Part string `json:"part"` + State string `json:"state"` + // Reason is why it is not healthy, in words with no address, path or domain; Said the detail. + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + // Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose + // file it is — the uplink's holder. + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + // Toward is what the failure points at: "hub", or each resolver's address that failed. + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` } // The states a long-running resource is said in (ADR 0240 §4). diff --git a/internal/link/queue.go b/internal/link/queue.go index a21ff72..6021f5f 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -479,6 +479,15 @@ func declaredIn(body []byte) string { return hex.EncodeToString(sum[:]) } +// Linked says whether a link to the bus is open now (novox/hq ADR 0241: the bus is one part of the +// machine's networking its engine judges). +func (q *Queue) Linked() bool { + q.init() + q.mu.Lock() + defer q.mu.Unlock() + return q.bus != nil +} + // SayHealth says a health statement on the link open now (novox/hq ADR 0240, to-be 48 §4), and whether // the bus took it. **Not through the worker**: a statement is a word about the machine as it is, not an // account of an apply, and it must not wait behind one — a container crash-looping while a long apply diff --git a/internal/network/dns.go b/internal/network/dns.go new file mode 100644 index 0000000..9c525e1 --- /dev/null +++ b/internal/network/dns.go @@ -0,0 +1,155 @@ +package network + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "net" + "strings" + "time" +) + +// The record types a look asks for. +const ( + TypeA uint16 = 1 + TypeAAAA uint16 = 28 +) + +// The answers a resolver gives that a look tells apart. +const ( + RcodeOK = 0 + RcodeServFail = 2 + RcodeNXDomain = 3 + RcodeRefused = 5 +) + +// Answer is what one resolver said to one question: its code, how many records of the type asked it +// gave, and how long it took. A question with no answer at all is an error, never an Answer. +type Answer struct { + Rcode int + Records int + Took time.Duration +} + +// Ask asks one resolver one question over UDP, and reads its code and how many records of the type +// asked it answered with. **It tells "no such name" from "no record of that type"** — the C library's +// lookup does not, and that difference is issue 262: an Alpine container failed a mesh name because a +// resolver said NXDOMAIN for its IPv6 address where it should have said there was none. +func Ask(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (Answer, error) { + start := time.Now() + query, id, err := question(name, qtype) + if err != nil { + return Answer{}, err + } + if net.ParseIP(server) != nil { + server = net.JoinHostPort(server, "53") + } + d := net.Dialer{Timeout: timeout} + conn, err := d.DialContext(ctx, "udp", server) + if err != nil { + return Answer{}, err + } + defer conn.Close() + _ = conn.SetDeadline(start.Add(timeout)) + if _, err := conn.Write(query); err != nil { + return Answer{}, err + } + buf := make([]byte, 1500) + for { + n, err := conn.Read(buf) + if err != nil { + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return Answer{}, fmt.Errorf("no answer within %s", timeout) + } + return Answer{}, err + } + a, ours, err := parse(buf[:n], id, qtype) + if !ours { + continue // a late answer to somebody else's question on this port + } + if err != nil { + return Answer{}, err + } + a.Took = time.Since(start) + return a, nil + } +} + +// question is one query: a header asking for recursion, and the name and type. +func question(name string, qtype uint16) ([]byte, uint16, error) { + var idb [2]byte + if _, err := rand.Read(idb[:]); err != nil { + return nil, 0, err + } + id := binary.BigEndian.Uint16(idb[:]) + msg := make([]byte, 12, 64) + binary.BigEndian.PutUint16(msg[0:], id) + msg[2] = 0x01 // recursion desired + binary.BigEndian.PutUint16(msg[4:], 1) + for _, label := range strings.Split(strings.TrimSuffix(name, "."), ".") { + if label == "" || len(label) > 63 { + return nil, 0, fmt.Errorf("%q is not a name that can be asked", name) + } + msg = append(msg, byte(len(label))) + msg = append(msg, label...) + } + msg = append(msg, 0, byte(qtype>>8), byte(qtype), 0, 1) + return msg, id, nil +} + +// parse reads an answer: false when it is not the answer to this question. +func parse(msg []byte, id, qtype uint16) (Answer, bool, error) { + if len(msg) < 12 || binary.BigEndian.Uint16(msg[0:]) != id || msg[2]&0x80 == 0 { + return Answer{}, false, nil + } + a := Answer{Rcode: int(msg[3] & 0x0f)} + qd, an := int(binary.BigEndian.Uint16(msg[4:])), int(binary.BigEndian.Uint16(msg[6:])) + at := 12 + for i := 0; i < qd; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return a, true, err + } + at += 4 + } + for i := 0; i < an; i++ { + var err error + if at, err = skipName(msg, at); err != nil { + return a, true, err + } + if at+10 > len(msg) { + return a, true, errors.New("the answer is cut short") + } + typ := binary.BigEndian.Uint16(msg[at:]) + length := int(binary.BigEndian.Uint16(msg[at+8:])) + at += 10 + length + if at > len(msg) { + return a, true, errors.New("the answer is cut short") + } + if typ == qtype { + a.Records++ + } + } + return a, true, nil +} + +// skipName steps over a name, compressed or not. +func skipName(msg []byte, at int) (int, error) { + for { + if at >= len(msg) { + return 0, errors.New("the answer is cut short") + } + l := int(msg[at]) + switch { + case l == 0: + return at + 1, nil + case l&0xc0 == 0xc0: + return at + 2, nil + default: + at += 1 + l + } + } +} diff --git a/internal/network/network.go b/internal/network/network.go new file mode 100644 index 0000000..5d46010 --- /dev/null +++ b/internal/network/network.go @@ -0,0 +1,598 @@ +// Package network is the node-engine judging its own machine's networking (novox/hq ADR 0241, which +// extends ADR 0240 from what a module runs to the machine it runs on). +// +// **A machine whose names stopped resolving read healthy.** On the laptop a corporate VPN client rewrote +// `/etc/resolv.conf` when it connected, replacing the mesh's resolvers (ADR 0223) with its own: mesh names +// failed, sometimes public ones too, and agents saw "no such host" for the services they call. The +// node-engine wrote the file back at its next reconcile, the client rewrote it again, and nothing said so — +// every module's own check was green, because no check asked the machine. A resolver slow under load +// (issue 277) and the tunnel to the hub are the same kind of fact: about the machine, under every module. +// +// Every LookEvery the judge looks at five parts, each cheaply: +// +// - **resolv-conf**: the file is what the uplink holder declared (ADR 0117, ADR 0223). Rewritten by +// another program, it says so — naming the program where the file, its link or what runs shows it; +// - **names**: every resolver the file lists answers a mesh name with an address and its IPv6 question +// with "none" rather than "no such name" (issue 262), and a public name with an address, within the +// time the file itself tells the C library to wait; +// - **tunnel**: the mesh's interface has a fresh handshake with the hub — on the hub, with any machine; +// - **bus**: the link to the bus is open; +// - **route**: the machine has a default route. +// +// **The two-look rule** (issue 277): a part is said unhealthy on its second failing look in a row, and +// healthy again on its first passing one. One unanswered datagram is not a finding. +// +// **It reads; it never acts** (ADR 0240 rule 6): nothing here writes the file back, restarts a link or +// asks a reconcile. The reconcile holds the file as it always has; this says when somebody else holds it. +package network + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// The bounds. +const ( + // LookEvery is how often the parts are looked at: about six datagrams per resolver and a read of + // three small files a minute, two looks to a finding inside the gate's first judging. + LookEvery = 30 * time.Second + // Confirm is how many failing looks in a row make a part unhealthy (issue 277). + Confirm = 2 + // StaleHandshake is how old the newest handshake with the hub may be. WireGuard renews a session + // every two minutes while anything passes over it, and the bus pings every two: past five, nothing + // has passed over the tunnel. + StaleHandshake = 5 * time.Minute + // ResolvConf is the file the uplink holder writes. + ResolvConf = "/etc/resolv.conf" + // PublicName is the public name asked: reserved for exactly this kind of use, answered by every + // public resolver, and belonging to no installation. + PublicName = "example.com" + // Interface is the mesh's own tunnel. + Interface = "mesh0" +) + +// The parts. +const ( + PartResolvConf = "resolv-conf" + PartNames = "names" + PartTunnel = "tunnel" + PartBus = "bus" + PartRoute = "route" +) + +// Parts is every part, in the order a person reads them. +var Parts = []string{PartResolvConf, PartNames, PartTunnel, PartBus, PartRoute} + +// The states, the words liveness says them in. +const ( + Healthy = "healthy" + Unhealthy = "unhealthy" + Unknown = "unknown" +) + +// TowardHub is what a part that fails toward the hub names: the tunnel and the bus. +const TowardHub = "hub" + +// Finding is one look at one part. +type Finding struct { + // Skip says the part is not judged on this machine: nothing declares the file, there is no tunnel. + Skip bool + OK bool + // Reason is why it is not healthy, in words that carry no address, path or name with its domain: + // it may reach the operator's channel. Said is the detail, which stays inside the mesh. + Reason string + Said string + // Writer is the program that rewrote the file, when the file, its link or what runs shows it. + Writer string + // Toward is what the failure points at: TowardHub, or each resolver's address that failed. + Toward []string +} + +// Part is one part's state, as the statement says it. +type Part struct { + Part string `json:"part"` + State string `json:"state"` + Reason string `json:"reason,omitempty"` + Said string `json:"said,omitempty"` + Writer string `json:"writer,omitempty"` + Owner string `json:"owner,omitempty"` + Toward []string `json:"toward,omitempty"` + Since time.Time `json:"since"` + Streak int `json:"streak,omitempty"` +} + +// Statement is one look at the machine's networking: the worst of its parts, since when, and each part. +type Statement struct { + State string `json:"state"` + Since time.Time `json:"since"` + At time.Time `json:"at"` + Parts []Part `json:"parts"` +} + +// Machine is what a look reads, replaced in tests. +type Machine struct { + // ResolvPath and ProcNet are where the file and the routing tables are. + ResolvPath string + ProcNet string + // Ask asks one resolver one question. + Ask func(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (Answer, error) + // Run runs a command: `wg`, to read the tunnel. + Run func(ctx context.Context, name string, args ...string) (string, error) + // Linked says whether the link to the bus is open now. + Linked func() bool + // Running is the names of the programs running, to name a writer by. Nil reads /proc. + Running func() []string + Now func() time.Time +} + +// declared is the file as the uplink holder declared it. +type declared struct { + content string + owner string +} + +type kept struct { + state string + since time.Time + streak int + last Finding +} + +// Judge is the one judge of this machine's networking. Safe for the apply and the looking loop at once. +type Judge struct { + m Machine + // MeshName is a name only the mesh's resolvers answer: the bus's own, which this machine needs most. + MeshName string + + mu sync.Mutex + file *declared + kept map[string]*kept + said Statement + lookedAt time.Time + overall kept +} + +// New is a judge of this machine, asking meshName as the mesh's name (empty when the bus is reached by +// address, and then no mesh name is asked). +func New(m Machine, meshName string) *Judge { + if m.ResolvPath == "" { + m.ResolvPath = ResolvConf + } + if m.ProcNet == "" { + m.ProcNet = "/proc/net" + } + if m.Ask == nil { + m.Ask = Ask + } + if m.Running == nil { + m.Running = running + } + if m.Now == nil { + m.Now = time.Now + } + return &Judge{m: m, MeshName: meshName, kept: map[string]*kept{}} +} + +// Declare is the file the uplink holder declared, from the declaration the apply just applied, and the +// module that declared it; ok false when nothing declares it whole, and then the file is not judged. +func (j *Judge) Declare(content, owner string, ok bool) { + j.mu.Lock() + defer j.mu.Unlock() + if !ok { + j.file = nil + return + } + j.file = &declared{content: content, owner: owner} +} + +// Look looks again when a look is due, and answers the statement and whether anything changed since the +// last; between looks it answers the last statement, unchanged. +func (j *Judge) Look(ctx context.Context) (Statement, bool) { + j.mu.Lock() + defer j.mu.Unlock() + now := j.m.Now() + if !j.lookedAt.IsZero() && now.Sub(j.lookedAt) < LookEvery { + return j.said, false + } + j.lookedAt = now + findings := map[string]Finding{ + PartResolvConf: j.lookFile(), + PartNames: j.lookNames(ctx), + PartTunnel: j.lookTunnel(ctx, now), + PartBus: j.lookBus(), + PartRoute: j.lookRoute(), + } + st := Statement{At: now, Parts: []Part{}} + changed := false + worst := Healthy + for _, name := range Parts { + f := findings[name] + k := j.kept[name] + if f.Skip { + if k != nil { + delete(j.kept, name) + changed = true + } + continue + } + if k == nil { + k = &kept{state: Unknown} + j.kept[name] = k + } + before := k.state + if f.OK { + k.streak = 0 + if k.state != Healthy { + k.state, k.since = Healthy, now + } + } else { + k.streak++ + if k.streak >= Confirm && k.state != Unhealthy { + k.state, k.since = Unhealthy, now + } + } + k.last = f + changed = changed || before != k.state + p := Part{Part: name, State: k.state, Since: k.since, Streak: k.streak} + if k.state == Unhealthy { + p.Reason, p.Said, p.Writer, p.Toward = f.Reason, f.Said, f.Writer, f.Toward + if name == PartResolvConf && j.file != nil { + p.Owner = j.file.owner + } + } + if k.state == Unknown && k.streak > 0 { + // Failing once: not yet a finding, and said as not known rather than as healthy. + p.Reason = "one look failed; a second decides" + } + st.Parts = append(st.Parts, p) + switch { + case k.state == Unhealthy: + worst = Unhealthy + case k.state == Unknown && worst == Healthy: + worst = Unknown + } + } + if j.overall.state != worst || j.overall.since.IsZero() { + j.overall.state, j.overall.since = worst, now + changed = true + } + st.State, st.Since = worst, j.overall.since + j.said = st + return st, changed +} + +// Last is the statement said last, without looking. +func (j *Judge) Last() Statement { + j.mu.Lock() + defer j.mu.Unlock() + return j.said +} + +// lookFile compares the file with what the uplink holder declared. +func (j *Judge) lookFile() Finding { + if j.file == nil { + return Finding{Skip: true} + } + path := j.m.ResolvPath + info, err := os.Lstat(path) + if err != nil { + return Finding{Reason: "the resolver file is missing", Said: err.Error(), Toward: nil} + } + if info.Mode()&os.ModeSymlink != 0 { + target, _ := os.Readlink(path) + return Finding{Reason: "the resolver file was replaced by a link, so another program now writes it", + Said: fmt.Sprintf("%s is a link to %s, not the file %s declares", path, target, j.file.owner), + Writer: writerOfLink(target)} + } + raw, err := os.ReadFile(path) + if err != nil { + return Finding{Reason: "the resolver file cannot be read", Said: err.Error()} + } + if strings.TrimSpace(string(raw)) == strings.TrimSpace(j.file.content) { + return Finding{OK: true} + } + writer, why := j.writerOf(string(raw), info.ModTime()) + said := fmt.Sprintf("%s differs from what %s declares: it lists %s where %s is declared; changed %s", + path, j.file.owner, listOrNone(nameservers(string(raw))), listOrNone(nameservers(j.file.content)), + info.ModTime().UTC().Format(time.RFC3339)) + if why != "" { + said += "; " + why + } + return Finding{Reason: "the resolver file was rewritten by another program", Said: said, Writer: writer} +} + +// lookNames asks every resolver the file lists — as the machine's programs read it now, whoever wrote it. +func (j *Judge) lookNames(ctx context.Context) Finding { + raw, err := os.ReadFile(j.m.ResolvPath) + if err != nil { + return Finding{Reason: "no resolver can be read from the resolver file", Said: err.Error()} + } + servers := nameservers(string(raw)) + if len(servers) == 0 { + return Finding{Reason: "the resolver file lists no resolver", Said: j.m.ResolvPath + " lists no nameserver"} + } + if len(servers) > 3 { + servers = servers[:3] // the C library reads three + } + bound := waitOf(string(raw)) + type question struct { + name string + qtype uint16 + mesh bool + } + var questions []question + if j.MeshName != "" { + questions = append(questions, question{j.MeshName, TypeA, true}, question{j.MeshName, TypeAAAA, true}) + } + questions = append(questions, question{PublicName, TypeA, false}) + + // Every question at once, so a look takes one bound however many resolvers are silent. + type result struct { + answer Answer + err error + } + results := make([][]result, len(servers)) + var wg sync.WaitGroup + for si, server := range servers { + results[si] = make([]result, len(questions)) + for qi, q := range questions { + wg.Add(1) + go func() { + defer wg.Done() + a, err := j.m.Ask(ctx, server, q.name, q.qtype, bound) + results[si][qi] = result{a, err} + }() + } + } + wg.Wait() + + var failing, said []string + meshFails, publicFails := 0, 0 + for si, server := range servers { + var wrong []string + for qi, q := range questions { + a, err := results[si][qi].answer, results[si][qi].err + word := "" + switch { + case err != nil: + word = err.Error() + case q.qtype == TypeAAAA: + // The mesh's names carry no IPv6 address: "none", never "no such name" (issue 262). + if a.Rcode != RcodeOK { + word = "says " + rcodeWords(a.Rcode) + " for its IPv6 address, where it should say there is none" + } + case a.Rcode != RcodeOK: + word = "says " + rcodeWords(a.Rcode) + case a.Records == 0: + word = "answers no address" + } + if word == "" { + continue + } + wrong = append(wrong, fmt.Sprintf("%s %s: %s", q.name, typeWords(q.qtype), word)) + if q.mesh { + meshFails++ + } else { + publicFails++ + } + } + if len(wrong) > 0 { + failing = append(failing, server) + said = append(said, server+" — "+strings.Join(wrong, "; ")) + } + } + if len(failing) == 0 { + return Finding{OK: true} + } + var reason string + switch { + case len(failing) < len(servers): + reason = fmt.Sprintf("%d of its %d resolvers do not answer as the mesh's do", len(failing), len(servers)) + case meshFails > 0 && publicFails > 0: + reason = "neither mesh names nor public names resolve" + case meshFails > 0: + reason = "mesh names do not resolve" + default: + reason = "public names do not resolve" + } + return Finding{Reason: reason, Said: fmt.Sprintf("within %s: %s", bound, strings.Join(said, " | ")), Toward: failing} +} + +// lookTunnel reads the mesh's interface: on a machine reaching the hub, the hub's handshake; on the hub, +// whether any machine has handshaken with it. +func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { + if j.m.Run == nil { + return Finding{Skip: true} + } + hs, err := j.m.Run(ctx, "wg", "show", Interface, "latest-handshakes") + if err != nil { + if strings.Contains(err.Error(), "executable file not found") { + return Finding{Skip: true} + } + return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()), + Toward: []string{TowardHub}} + } + ips, err := j.m.Run(ctx, "wg", "show", Interface, "allowed-ips") + if err != nil { + return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()), + Toward: []string{TowardHub}} + } + handshakes := map[string]int64{} + for _, line := range strings.Split(hs, "\n") { + f := strings.Fields(line) + if len(f) == 2 { + at, _ := strconv.ParseInt(f[1], 10, 64) + handshakes[f[0]] = at + } + } + hub := "" + for _, line := range strings.Split(ips, "\n") { + f := strings.Fields(line) + for _, prefix := range f[min(1, len(f)):] { + if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" { + hub = f[0] + } + } + } + age := func(at int64) string { + if at == 0 { + return "never" + } + return now.Sub(time.Unix(at, 0)).Round(time.Second).String() + " ago" + } + if hub != "" { + at := handshakes[hub] + if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake { + return Finding{OK: true} + } + return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}} + } + if len(handshakes) == 0 { + return Finding{OK: true} + } + var newest int64 + for _, at := range handshakes { + newest = max(newest, at) + } + if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake { + return Finding{OK: true} + } + return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))} +} + +func (j *Judge) lookBus() Finding { + if j.m.Linked == nil { + return Finding{Skip: true} + } + if j.m.Linked() { + return Finding{OK: true} + } + return Finding{Reason: "the bus cannot be reached", Said: "no link to the bus is open", Toward: []string{TowardHub}} +} + +func (j *Judge) lookRoute() Finding { + var routes []string + for _, table := range []struct { + file string + dest, mask, i int + }{{"route", 1, 7, 0}, {"ipv6_route", 0, 1, 9}} { + f, err := os.Open(filepath.Join(j.m.ProcNet, table.file)) + if err != nil { + continue + } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) <= max(table.dest, table.mask, table.i) || fields[0] == "Iface" { + continue + } + if zero(fields[table.dest]) && zero(fields[table.mask]) && fields[table.i] != "lo" { + routes = append(routes, fields[table.i]) + } + } + f.Close() + } + if len(routes) > 0 { + return Finding{OK: true} + } + return Finding{Reason: "the machine has no default route", Said: "no default route in " + j.m.ProcNet} +} + +// nameservers is every resolver a file lists, in order. +func nameservers(content string) []string { + var out []string + for _, line := range strings.Split(content, "\n") { + f := strings.Fields(line) + if len(f) >= 2 && f[0] == "nameserver" { + out = append(out, f[1]) + } + } + return out +} + +// waitOf is how long the file tells the C library to wait for one resolver: `options timeout:n`, five +// seconds when it says nothing, and never under one. +func waitOf(content string) time.Duration { + wait := 5 * time.Second + for _, line := range strings.Split(content, "\n") { + f := strings.Fields(line) + if len(f) == 0 || f[0] != "options" { + continue + } + for _, o := range f[1:] { + if v, ok := strings.CutPrefix(o, "timeout:"); ok { + if n, err := strconv.Atoi(v); err == nil { + wait = time.Duration(max(n, 1)) * time.Second + } + } + } + } + return wait +} + +func rcodeWords(rcode int) string { + switch rcode { + case RcodeNXDomain: + return "no such name" + case RcodeServFail: + return "it failed" + case RcodeRefused: + return "it refuses" + } + return fmt.Sprintf("code %d", rcode) +} + +func typeWords(t uint16) string { + if t == TypeAAAA { + return "(IPv6)" + } + return "(IPv4)" +} + +func listOrNone(s []string) string { + if len(s) == 0 { + return "no resolver" + } + return strings.Join(s, ", ") +} + +func zero(hex string) bool { return strings.Trim(hex, "0") == "" } + +func firstLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} + +// running is the names of the programs running, from /proc. +func running() []string { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil + } + seen := map[string]bool{} + for _, e := range entries { + if _, err := strconv.Atoi(e.Name()); err != nil { + continue + } + comm, err := os.ReadFile(filepath.Join("/proc", e.Name(), "comm")) + if err == nil { + seen[strings.TrimSpace(string(comm))] = true + } + } + out := make([]string, 0, len(seen)) + for n := range seen { + out = append(out, n) + } + sort.Strings(out) + return out +} diff --git a/internal/network/network_test.go b/internal/network/network_test.go new file mode 100644 index 0000000..f27caa0 --- /dev/null +++ b/internal/network/network_test.go @@ -0,0 +1,406 @@ +package network + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// novox/hq ADR 0241, "how it is checked": the file rewritten by another program is said on the second +// look, naming the writer; written back, healthy on the first; a resolver answering NXDOMAIN for a mesh +// name's IPv6 address is a finding (issue 262); a stale handshake with the hub, the bus unlinked and no +// default route are each said; one failing look is not a finding. + +const meshFile = `# Managed by the mesh, and written by the module holding this machine's uplink. +nameserver 10.10.0.2 +nameserver 10.10.0.1 +options timeout:1 attempts:2 edns0 +` + +// vpnFile is what the VPN client writes on connect, its header as it writes it. +const vpnFile = `# Dynamic resolv.conf(5) file for glibc resolver(3) generated by forticlient +# The original file is backed up and will be restored after the VPN disconnects. +nameserver 172.16.5.5 +nameserver 172.16.5.6 +search corp.example +` + +type fakeMachine struct { + dir string + now time.Time + linked bool + answers map[string]Answer // server|name|type + wrong map[string]error + hs, ips string + wgErr error + running []string +} + +func newFake(t *testing.T) *fakeMachine { + t.Helper() + dir := t.TempDir() + f := &fakeMachine{dir: dir, now: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC), linked: true, + answers: map[string]Answer{}, wrong: map[string]error{}} + f.write(t, meshFile) + if err := os.MkdirAll(filepath.Join(dir, "net"), 0o755); err != nil { + t.Fatal(err) + } + f.route(t, true) + f.hub(f.now.Add(-time.Minute)) + return f +} + +func (f *fakeMachine) write(t *testing.T, content string) { + t.Helper() + path := filepath.Join(f.dir, "resolv.conf") + os.Remove(path) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func (f *fakeMachine) route(t *testing.T, has bool) { + t.Helper() + table := "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\n" + + "mesh0\t00000A0A\t00000000\t0001\t0\t0\t0\t00FFFFFF\n" + if has { + table += "wlan0\t00000000\t0101A8C0\t0003\t0\t0\t600\t00000000\n" + } + if err := os.WriteFile(filepath.Join(f.dir, "net", "route"), []byte(table), 0o644); err != nil { + t.Fatal(err) + } +} + +// hub is a machine reaching the hub, its newest handshake at at. +func (f *fakeMachine) hub(at time.Time) { + f.hs = "HUBKEY=\t" + itoa(at.Unix()) + "\n" + f.ips = "HUBKEY=\t10.10.0.0/24\n" +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } + +func (f *fakeMachine) judge(t *testing.T) *Judge { + t.Helper() + j := New(Machine{ + ResolvPath: filepath.Join(f.dir, "resolv.conf"), + ProcNet: filepath.Join(f.dir, "net"), + Ask: func(_ context.Context, server, name string, qtype uint16, _ time.Duration) (Answer, error) { + key := server + "|" + name + "|" + typeWords(qtype) + if err, ok := f.wrong[key]; ok { + return Answer{}, err + } + if a, ok := f.answers[key]; ok { + return a, nil + } + switch { + case strings.HasPrefix(server, "10.10.") && qtype == TypeAAAA: + return Answer{}, nil // the mesh's names have no IPv6 address: none, not no such name + case strings.HasPrefix(server, "10.10."): + return Answer{Records: 1}, nil + case name == "novox.internal": + return Answer{Rcode: RcodeNXDomain}, nil // the VPN's resolver knows no mesh name + } + return Answer{Records: 1}, nil + }, + Run: func(_ context.Context, name string, args ...string) (string, error) { + if f.wgErr != nil { + return "", f.wgErr + } + if args[len(args)-1] == "latest-handshakes" { + return f.hs, nil + } + return f.ips, nil + }, + Linked: func() bool { return f.linked }, + Running: func() []string { return f.running }, + Now: func() time.Time { return f.now }, + }, "novox.internal") + j.Declare(meshFile, "networkmanager", true) + return j +} + +// look moves the clock a look on and looks. +func (f *fakeMachine) look(t *testing.T, j *Judge) Statement { + t.Helper() + f.now = f.now.Add(LookEvery) + st, _ := j.Look(t.Context()) + return st +} + +func partOf(st Statement, name string) (Part, bool) { + for _, p := range st.Parts { + if p.Part == name { + return p, true + } + } + return Part{}, false +} + +func TestAHealthyMachineIsSaidHealthyOnItsFirstLook(t *testing.T) { + f := newFake(t) + j := f.judge(t) + st := f.look(t, j) + if st.State != Healthy { + t.Fatalf("a healthy machine is said %s: %+v", st.State, st.Parts) + } + if len(st.Parts) != len(Parts) { + t.Fatalf("want every part judged, got %+v", st.Parts) + } +} + +func TestAFileRewrittenByAVPNClientIsSaidOnTheSecondLookNamingItAndClearedWhenWrittenBack(t *testing.T) { + f := newFake(t) + j := f.judge(t) + f.look(t, j) + f.write(t, vpnFile) + + st := f.look(t, j) + if st.State == Unhealthy { + t.Fatalf("one look raised it: %+v", st.Parts) + } + if p, _ := partOf(st, PartResolvConf); p.State != Healthy || p.Streak != 1 { + t.Fatalf("after one failing look the file is %+v; want still healthy, a streak of one", p) + } + + st = f.look(t, j) + if st.State != Unhealthy { + t.Fatalf("two looks did not make it unhealthy: %+v", st.Parts) + } + p, _ := partOf(st, PartResolvConf) + if p.State != Unhealthy || p.Writer != "FortiClient" || p.Owner != "networkmanager" { + t.Fatalf("the file is said %+v; want unhealthy, written by FortiClient, owned by networkmanager", p) + } + if strings.Contains(p.Reason, "172.16.") || !strings.Contains(p.Said, "172.16.5.5") { + t.Fatalf("the addresses belong in what is said, never the reason: %+v", p) + } + // And the mesh's names do not resolve through what the VPN client wrote. + names, _ := partOf(st, PartNames) + if names.State != Unhealthy || names.Reason != "mesh names do not resolve" { + t.Fatalf("names through the VPN's resolvers are said %+v", names) + } + + f.write(t, meshFile) + st = f.look(t, j) + if st.State != Healthy { + t.Fatalf("written back, it is still %s: %+v", st.State, st.Parts) + } +} + +func TestAWriterIsNamedByWhatRunsWhenTheFileSaysNothing(t *testing.T) { + f := newFake(t) + f.running = []string{"systemd", "openvpn"} + j := f.judge(t) + f.write(t, "nameserver 192.0.2.53\n") + f.look(t, j) + st := f.look(t, j) + p, _ := partOf(st, PartResolvConf) + if p.Writer != "OpenVPN?" || !strings.Contains(p.Said, "openvpn is running") { + t.Fatalf("want the running VPN client named as a guess, got %+v", p) + } +} + +func TestALinkInPlaceOfTheFileNamesWhatItPointsAt(t *testing.T) { + f := newFake(t) + j := f.judge(t) + target := filepath.Join(f.dir, "stub-resolv.conf") + if err := os.WriteFile(target, []byte(meshFile), 0o644); err != nil { + t.Fatal(err) + } + path := filepath.Join(f.dir, "systemd", "resolve") + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Rename(target, filepath.Join(path, "stub-resolv.conf")); err != nil { + t.Fatal(err) + } + os.Remove(filepath.Join(f.dir, "resolv.conf")) + if err := os.Symlink(filepath.Join(path, "stub-resolv.conf"), filepath.Join(f.dir, "resolv.conf")); err != nil { + t.Fatal(err) + } + f.look(t, j) + st := f.look(t, j) + p, _ := partOf(st, PartResolvConf) + if p.State != Unhealthy || p.Writer != "systemd-resolved" { + t.Fatalf("a link is said %+v", p) + } +} + +func TestNothingDeclaredIsNotJudged(t *testing.T) { + f := newFake(t) + j := f.judge(t) + j.Declare("", "", false) + f.write(t, vpnFile) + f.look(t, j) + st := f.look(t, j) + if _, judged := partOf(st, PartResolvConf); judged { + t.Fatalf("a file nothing declares was judged: %+v", st.Parts) + } +} + +func TestNXDomainForAMeshNamesIPv6AddressIsAFinding(t *testing.T) { + f := newFake(t) + f.answers["10.10.0.1|novox.internal|(IPv6)"] = Answer{Rcode: RcodeNXDomain} + j := f.judge(t) + f.look(t, j) + st := f.look(t, j) + p, _ := partOf(st, PartNames) + if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.1" || + p.Reason != "1 of its 2 resolvers do not answer as the mesh's do" || !strings.Contains(p.Said, "IPv6") { + t.Fatalf("issue 262's answer is said %+v", p) + } +} + +func TestAResolverThatDoesNotAnswerIsNamedAndOneLookIsNotAFinding(t *testing.T) { + f := newFake(t) + j := f.judge(t) + f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s") + st := f.look(t, j) + if p, _ := partOf(st, PartNames); p.State == Unhealthy { + t.Fatalf("one unanswered question raised it: %+v", p) + } + delete(f.wrong, "10.10.0.2|novox.internal|(IPv4)") + if st := f.look(t, j); st.State != Healthy { + t.Fatalf("answered again, it is %s", st.State) + } + f.wrong["10.10.0.2|novox.internal|(IPv4)"] = errors.New("no answer within 1s") + f.look(t, j) + st = f.look(t, j) + p, _ := partOf(st, PartNames) + if p.State != Unhealthy || len(p.Toward) != 1 || p.Toward[0] != "10.10.0.2" { + t.Fatalf("a silent resolver is said %+v", p) + } +} + +func TestTheTunnelTheBusAndTheRouteAreEachSaid(t *testing.T) { + f := newFake(t) + j := f.judge(t) + f.hub(f.now.Add(-10 * time.Minute)) + f.linked = false + f.route(t, false) + f.look(t, j) + st := f.look(t, j) + for _, name := range []string{PartTunnel, PartBus, PartRoute} { + p, _ := partOf(st, name) + if p.State != Unhealthy { + t.Fatalf("%s is said %+v", name, p) + } + } + if p, _ := partOf(st, PartTunnel); len(p.Toward) != 1 || p.Toward[0] != TowardHub { + t.Fatalf("the tunnel's failure does not point at the hub: %+v", p) + } +} + +func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) { + f := newFake(t) + f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n" + f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n" + j := f.judge(t) + if st := f.look(t, j); st.State != Healthy { + t.Fatalf("the hub with one fresh peer is %+v", st.Parts) + } +} + +func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) { + f := newFake(t) + f.wgErr = errors.New(`exec: "wg": executable file not found in $PATH`) + j := f.judge(t) + st := f.look(t, j) + if _, judged := partOf(st, PartTunnel); judged { + t.Fatal("a machine without wg judged a tunnel") + } +} + +func TestBetweenLooksTheLastStatementIsAnswered(t *testing.T) { + f := newFake(t) + var calls atomic.Int64 + j := f.judge(t) + ask := j.m.Ask + j.m.Ask = func(ctx context.Context, s, n string, q uint16, d time.Duration) (Answer, error) { + calls.Add(1) + return ask(ctx, s, n, q, d) + } + f.look(t, j) + before := calls.Load() + f.now = f.now.Add(LookEvery / 2) + if _, changed := j.Look(t.Context()); changed || calls.Load() != before { + t.Fatalf("a look half a period later asked again (%d questions) or said a change", calls.Load()-before) + } +} + +func TestTheWaitIsTheFilesOwn(t *testing.T) { + if w := waitOf(meshFile); w != time.Second { + t.Fatalf("timeout:1 is %s", w) + } + if w := waitOf("nameserver 192.0.2.1\n"); w != 5*time.Second { + t.Fatalf("no options is %s", w) + } +} + +// TestAskReadsARealAnswer asks a resolver this test raises: an address for one name, none for its IPv6 +// address, and no such name for another. +func TestAskReadsARealAnswer(t *testing.T) { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Skip("no UDP here:", err) + } + defer pc.Close() + go func() { + buf := make([]byte, 512) + for { + n, from, err := pc.ReadFrom(buf) + if err != nil { + return + } + q := append([]byte(nil), buf[:n]...) + resp := append([]byte(nil), q...) + resp[2] |= 0x80 + qtype := uint16(q[n-4])<<8 | uint16(q[n-3]) + switch { + case strings.Contains(string(q), "missing"): + resp[3] = RcodeNXDomain + case qtype == TypeA: + resp[7] = 1 + resp = append(resp, 0xc0, 12, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4, 10, 10, 0, 1) + } + pc.WriteTo(resp, from) + } + }() + server := pc.LocalAddr().String() + ctx := t.Context() + if a, err := Ask(ctx, server, "novox.internal", TypeA, time.Second); err != nil || a.Rcode != 0 || a.Records != 1 { + t.Fatalf("A: %+v %v", a, err) + } + if a, err := Ask(ctx, server, "novox.internal", TypeAAAA, time.Second); err != nil || a.Rcode != 0 || a.Records != 0 { + t.Fatalf("AAAA: %+v %v", a, err) + } + if a, err := Ask(ctx, server, "missing.internal", TypeA, time.Second); err != nil || a.Rcode != RcodeNXDomain { + t.Fatalf("NXDOMAIN: %+v %v", a, err) + } + if _, err := Ask(ctx, "127.0.0.1:9", "novox.internal", TypeA, 200*time.Millisecond); err == nil { + t.Fatal("a resolver that does not answer answered") + } +} + +func TestABackupNamedForItsWriterNamesItWhateverTheFileSays(t *testing.T) { + f := newFake(t) + j := f.judge(t) + // The client renames the mesh's file aside: the backup keeps the old file's time. + if err := os.WriteFile(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), []byte(meshFile), 0o644); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-time.Hour) + os.Chtimes(filepath.Join(f.dir, "resolv.conf.forticlient.backup"), old, old) + f.write(t, "nameserver 192.0.2.53\n") + f.look(t, j) + st := f.look(t, j) + if p, _ := partOf(st, PartResolvConf); p.Writer != "FortiClient" || !strings.Contains(p.Said, "forticlient.backup") { + t.Fatalf("the backup did not name its writer: %+v", p) + } +} diff --git a/internal/network/writer.go b/internal/network/writer.go new file mode 100644 index 0000000..a5b5c72 --- /dev/null +++ b/internal/network/writer.go @@ -0,0 +1,119 @@ +package network + +import ( + "os" + "path/filepath" + "strings" + "time" +) + +// Naming the program that rewrote the resolver file (ADR 0241 rule 2). **A guess, said as one**: the +// mesh cannot see who wrote a file after the fact, only what the file, its link and the machine show. In +// order of how much each says: +// +// 1. what the file says of itself — every program that writes it puts its name in a comment; +// 2. a backup the writer left beside it — named for the writer, or changed when the file was; +// 3. a program known to write the file, running now. +// +// Nothing found is said as nothing found, never as a name. + +// signs are the words a writer leaves in the file's comments, and its name. +var signs = []struct{ word, name string }{ + {"forti", "FortiClient"}, + {"openfortivpn", "openfortivpn"}, + {"networkmanager", "NetworkManager"}, + {"systemd-resolved", "systemd-resolved"}, + {"resolvconf", "resolvconf"}, + {"dhcpcd", "dhcpcd"}, + {"dhclient", "dhclient"}, + {"netconfig", "netconfig"}, + {"openvpn", "OpenVPN"}, + {"openconnect", "OpenConnect"}, + {"vpnc", "vpnc"}, + {"tailscale", "Tailscale"}, + {"connman", "ConnMan"}, +} + +// writers are the programs known to rewrite the file, as they run, and their name. The VPN clients +// first: they are what rewrites a file the machine's network manager was already told to keep off. +var writers = []struct{ comm, name string }{ + {"fortivpn", "FortiClient"}, + {"forticlient", "FortiClient"}, + {"fctsched", "FortiClient"}, + {"openfortivpn", "openfortivpn"}, + {"openvpn", "OpenVPN"}, + {"openconnect", "OpenConnect"}, + {"vpnc", "vpnc"}, + {"charon", "strongSwan"}, + {"tailscaled", "Tailscale"}, + {"dhclient", "dhclient"}, + {"resolvconf", "resolvconf"}, +} + +// writerOf names who rewrote the file, and why that name, from the file's own words, a backup changed +// beside it, or a writer running. +func (j *Judge) writerOf(content string, changed time.Time) (string, string) { + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") && !strings.HasPrefix(line, ";") { + continue + } + lower := strings.ToLower(line) + for _, s := range signs { + if strings.Contains(lower, s.word) { + return s.name, "its own header names " + s.name + } + } + } + // A backup the writer kept beside it. + backup := "" + matches, _ := filepath.Glob(j.m.ResolvPath + "*") + for _, m := range matches { + if m == j.m.ResolvPath { + continue + } + info, err := os.Stat(m) + if err != nil || info.IsDir() { + continue + } + // A backup that names its writer says so whenever it was made: a client that renames the file + // aside (FortiClient does, on connect) leaves the backup with the old file's time, not its own. + lower := strings.ToLower(filepath.Base(m)) + for _, s := range signs { + if strings.Contains(lower, s.word) { + return s.name, "it left " + m + " beside it" + } + } + if d := info.ModTime().Sub(changed); d >= -time.Minute && d <= time.Minute { + backup = m + } + } + why := "no program it could be is known" + if backup != "" { + why = backup + " was changed when it was: whoever wrote it kept a copy there" + } + runningNow := map[string]bool{} + for _, name := range j.m.Running() { + runningNow[strings.ToLower(name)] = true + } + for _, w := range writers { + if runningNow[w.comm] { + return w.name + "?", w.comm + " is running; " + why + } + } + return "", why +} + +// writerOfLink names the program a link points the file at. +func writerOfLink(target string) string { + lower := strings.ToLower(target) + switch { + case strings.Contains(lower, "systemd/resolve"): + return "systemd-resolved" + case strings.Contains(lower, "resolvconf"): + return "resolvconf" + case strings.Contains(lower, "networkmanager"): + return "NetworkManager" + } + return "" +}