mesh-host/internal/network/network.go

// Package network is the node-engine judging its own machine's networking (novox/hq ADR 0241, which
// extends ADR 0240 from what a module runs to the machine it runs on).
//
// **A machine whose names stopped resolving read healthy.** On the laptop a corporate VPN client rewrote
// `/etc/resolv.conf` when it connected, replacing the mesh's resolvers (ADR 0223) with its own: mesh names
// failed, sometimes public ones too, and agents saw "no such host" for the services they call. The
// node-engine wrote the file back at its next reconcile, the client rewrote it again, and nothing said so —
// every module's own check was green, because no check asked the machine. A resolver slow under load
// (issue 277) and the tunnel to the hub are the same kind of fact: about the machine, under every module.
//
// Every LookEvery the judge looks at five parts, each cheaply:
//
//   - **resolv-conf**: the file is what the uplink holder declared (ADR 0117, ADR 0223). Rewritten by
//     another program, it says so — naming the program where the file, its link or what runs shows it;
//   - **names**: every resolver the file lists answers a mesh name with an address and its IPv6 question
//     with "none" rather than "no such name" (issue 262), and a public name with an address, within the
//     time the file itself tells the C library to wait;
//   - **tunnel**: the mesh's interface has a fresh handshake with the hub — on the hub, with any machine;
//   - **bus**: the link to the bus is open;
//   - **route**: the machine has a default route.
//
// **The two-look rule** (issue 277): a part is said unhealthy on its second failing look in a row, and
// healthy again on its first passing one. One unanswered datagram is not a finding.
//
// **It reads; it never acts** (ADR 0240 rule 6): nothing here writes the file back, restarts a link or
// asks a reconcile. The reconcile holds the file as it always has; this says when somebody else holds it.
package network

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// The bounds.
const (
	// LookEvery is how often the parts are looked at: about six datagrams per resolver and a read of
	// three small files a minute, two looks to a finding inside the gate's first judging.
	LookEvery = 30 * time.Second
	// Confirm is how many failing looks in a row make a part unhealthy (issue 277).
	Confirm = 2
	// StaleHandshake is how old the newest handshake with the hub may be. WireGuard renews a session
	// every two minutes while anything passes over it, and the bus pings every two: past five, nothing
	// has passed over the tunnel.
	StaleHandshake = 5 * time.Minute
	// ResolvConf is the file the uplink holder writes.
	ResolvConf = "/etc/resolv.conf"
	// PublicName is the public name asked: reserved for exactly this kind of use, answered by every
	// public resolver, and belonging to no installation.
	PublicName = "example.com"
	// Interface is the mesh's own tunnel.
	Interface = "mesh0"
)

// The parts.
const (
	PartResolvConf = "resolv-conf"
	PartNames      = "names"
	PartTunnel     = "tunnel"
	PartBus        = "bus"
	PartRoute      = "route"
)

// Parts is every part, in the order a person reads them.
var Parts = []string{PartResolvConf, PartNames, PartTunnel, PartBus, PartRoute}

// The states, the words liveness says them in.
const (
	Healthy   = "healthy"
	Unhealthy = "unhealthy"
	Unknown   = "unknown"
)

// TowardHub is what a part that fails toward the hub names: the tunnel and the bus.
const TowardHub = "hub"

// Finding is one look at one part.
type Finding struct {
	// Skip says the part is not judged on this machine: nothing declares the file, there is no tunnel.
	Skip bool
	OK   bool
	// Reason is why it is not healthy, in words that carry no address, path or name with its domain:
	// it may reach the operator's channel. Said is the detail, which stays inside the mesh.
	Reason string
	Said   string
	// Writer is the program that rewrote the file, when the file, its link or what runs shows it.
	Writer string
	// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
	Toward []string
}

// Part is one part's state, as the statement says it.
type Part struct {
	Part   string    `json:"part"`
	State  string    `json:"state"`
	Reason string    `json:"reason,omitempty"`
	Said   string    `json:"said,omitempty"`
	Writer string    `json:"writer,omitempty"`
	Owner  string    `json:"owner,omitempty"`
	Toward []string  `json:"toward,omitempty"`
	Since  time.Time `json:"since"`
	Streak int       `json:"streak,omitempty"`
}

// Statement is one look at the machine's networking: the worst of its parts, since when, and each part.
type Statement struct {
	State string    `json:"state"`
	Since time.Time `json:"since"`
	At    time.Time `json:"at"`
	Parts []Part    `json:"parts"`
}

// Machine is what a look reads, replaced in tests.
type Machine struct {
	// ResolvPath and ProcNet are where the file and the routing tables are.
	ResolvPath string
	ProcNet    string
	// Ask asks one resolver one question.
	Ask func(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (Answer, error)
	// Run runs a command: `wg`, to read the tunnel.
	Run func(ctx context.Context, name string, args ...string) (string, error)
	// Linked says whether the link to the bus is open now.
	Linked func() bool
	// Running is the names of the programs running, to name a writer by. Nil reads /proc.
	Running func() []string
	Now     func() time.Time
}

// declared is the file as the uplink holder declared it.
type declared struct {
	content string
	owner   string
}

type kept struct {
	state  string
	since  time.Time
	streak int
	last   Finding
}

// Judge is the one judge of this machine's networking. Safe for the apply and the looking loop at once.
type Judge struct {
	m Machine
	// MeshName is a name only the mesh's resolvers answer: the bus's own, which this machine needs most.
	MeshName string

	mu       sync.Mutex
	file     *declared
	kept     map[string]*kept
	said     Statement
	lookedAt time.Time
	overall  kept
}

// New is a judge of this machine, asking meshName as the mesh's name (empty when the bus is reached by
// address, and then no mesh name is asked).
func New(m Machine, meshName string) *Judge {
	if m.ResolvPath == "" {
		m.ResolvPath = ResolvConf
	}
	if m.ProcNet == "" {
		m.ProcNet = "/proc/net"
	}
	if m.Ask == nil {
		m.Ask = Ask
	}
	if m.Running == nil {
		m.Running = running
	}
	if m.Now == nil {
		m.Now = time.Now
	}
	return &Judge{m: m, MeshName: meshName, kept: map[string]*kept{}}
}

// Declare is the file the uplink holder declared, from the declaration the apply just applied, and the
// module that declared it; ok false when nothing declares it whole, and then the file is not judged.
func (j *Judge) Declare(content, owner string, ok bool) {
	j.mu.Lock()
	defer j.mu.Unlock()
	if !ok {
		j.file = nil
		return
	}
	j.file = &declared{content: content, owner: owner}
}

// Look looks again when a look is due, and answers the statement and whether anything changed since the
// last; between looks it answers the last statement, unchanged.
func (j *Judge) Look(ctx context.Context) (Statement, bool) {
	j.mu.Lock()
	defer j.mu.Unlock()
	now := j.m.Now()
	// A tick a little early is still the tick: the loop that calls this runs on its own clock.
	if !j.lookedAt.IsZero() && now.Sub(j.lookedAt) < LookEvery-LookEvery/10 {
		return j.said, false
	}
	j.lookedAt = now
	findings := map[string]Finding{
		PartResolvConf: j.lookFile(),
		PartNames:      j.lookNames(ctx),
		PartTunnel:     j.lookTunnel(ctx, now),
		PartBus:        j.lookBus(),
		PartRoute:      j.lookRoute(),
	}
	st := Statement{At: now, Parts: []Part{}}
	changed := false
	worst := Healthy
	for _, name := range Parts {
		f := findings[name]
		k := j.kept[name]
		if f.Skip {
			if k != nil {
				delete(j.kept, name)
				changed = true
			}
			continue
		}
		if k == nil {
			k = &kept{state: Unknown}
			j.kept[name] = k
		}
		before := k.state
		if f.OK {
			k.streak = 0
			if k.state != Healthy {
				k.state, k.since = Healthy, now
			}
		} else {
			k.streak++
			if k.streak >= Confirm && k.state != Unhealthy {
				k.state, k.since = Unhealthy, now
			}
		}
		k.last = f
		changed = changed || before != k.state
		p := Part{Part: name, State: k.state, Since: k.since, Streak: k.streak}
		if k.state == Unhealthy {
			p.Reason, p.Said, p.Writer, p.Toward = f.Reason, f.Said, f.Writer, f.Toward
			if name == PartResolvConf && j.file != nil {
				p.Owner = j.file.owner
			}
		}
		if k.state == Unknown && k.streak > 0 {
			// Failing once: not yet a finding, and said as not known rather than as healthy.
			p.Reason = "one look failed; a second decides"
		}
		st.Parts = append(st.Parts, p)
		switch {
		case k.state == Unhealthy:
			worst = Unhealthy
		case k.state == Unknown && worst == Healthy:
			worst = Unknown
		}
	}
	if j.overall.state != worst || j.overall.since.IsZero() {
		j.overall.state, j.overall.since = worst, now
		changed = true
	}
	st.State, st.Since = worst, j.overall.since
	j.said = st
	return st, changed
}

// Last is the statement said last, without looking.
func (j *Judge) Last() Statement {
	j.mu.Lock()
	defer j.mu.Unlock()
	return j.said
}

// lookFile compares the file with what the uplink holder declared.
func (j *Judge) lookFile() Finding {
	if j.file == nil {
		return Finding{Skip: true}
	}
	path := j.m.ResolvPath
	info, err := os.Lstat(path)
	if err != nil {
		return Finding{Reason: "the resolver file is missing", Said: err.Error(), Toward: nil}
	}
	if info.Mode()&os.ModeSymlink != 0 {
		target, _ := os.Readlink(path)
		return Finding{Reason: "the resolver file was replaced by a link, so another program now writes it",
			Said:   fmt.Sprintf("%s is a link to %s, not the file %s declares", path, target, j.file.owner),
			Writer: writerOfLink(target)}
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		return Finding{Reason: "the resolver file cannot be read", Said: err.Error()}
	}
	if strings.TrimSpace(string(raw)) == strings.TrimSpace(j.file.content) {
		return Finding{OK: true}
	}
	writer, why := j.writerOf(string(raw), info.ModTime())
	said := fmt.Sprintf("%s differs from what %s declares: it lists %s where %s is declared; changed %s",
		path, j.file.owner, listOrNone(nameservers(string(raw))), listOrNone(nameservers(j.file.content)),
		info.ModTime().UTC().Format(time.RFC3339))
	if why != "" {
		said += "; " + why
	}
	return Finding{Reason: "the resolver file was rewritten by another program", Said: said, Writer: writer}
}

// lookNames asks every resolver the file lists — as the machine's programs read it now, whoever wrote it.
func (j *Judge) lookNames(ctx context.Context) Finding {
	raw, err := os.ReadFile(j.m.ResolvPath)
	if err != nil {
		return Finding{Reason: "no resolver can be read from the resolver file", Said: err.Error()}
	}
	servers := nameservers(string(raw))
	if len(servers) == 0 {
		return Finding{Reason: "the resolver file lists no resolver", Said: j.m.ResolvPath + " lists no nameserver"}
	}
	if len(servers) > 3 {
		servers = servers[:3] // the C library reads three
	}
	bound := waitOf(string(raw))
	type question struct {
		name  string
		qtype uint16
		mesh  bool
	}
	var questions []question
	if j.MeshName != "" {
		questions = append(questions, question{j.MeshName, TypeA, true}, question{j.MeshName, TypeAAAA, true})
	}
	questions = append(questions, question{PublicName, TypeA, false})

	// Every question at once, so a look takes one bound however many resolvers are silent.
	type result struct {
		answer Answer
		err    error
	}
	results := make([][]result, len(servers))
	var wg sync.WaitGroup
	for si, server := range servers {
		results[si] = make([]result, len(questions))
		for qi, q := range questions {
			wg.Add(1)
			go func() {
				defer wg.Done()
				a, err := j.m.Ask(ctx, server, q.name, q.qtype, bound)
				results[si][qi] = result{a, err}
			}()
		}
	}
	wg.Wait()

	var failing, said []string
	meshFails, publicFails := 0, 0
	for si, server := range servers {
		var wrong []string
		for qi, q := range questions {
			a, err := results[si][qi].answer, results[si][qi].err
			word := ""
			switch {
			case err != nil:
				word = err.Error()
			case q.qtype == TypeAAAA:
				// The mesh's names carry no IPv6 address: "none", never "no such name" (issue 262).
				if a.Rcode != RcodeOK {
					word = "says " + rcodeWords(a.Rcode) + " for its IPv6 address, where it should say there is none"
				}
			case a.Rcode != RcodeOK:
				word = "says " + rcodeWords(a.Rcode)
			case a.Records == 0:
				word = "answers no address"
			}
			if word == "" {
				continue
			}
			wrong = append(wrong, fmt.Sprintf("%s %s: %s", q.name, typeWords(q.qtype), word))
			if q.mesh {
				meshFails++
			} else {
				publicFails++
			}
		}
		if len(wrong) > 0 {
			failing = append(failing, server)
			said = append(said, server+" — "+strings.Join(wrong, "; "))
		}
	}
	if len(failing) == 0 {
		return Finding{OK: true}
	}
	var reason string
	switch {
	case len(failing) < len(servers):
		reason = fmt.Sprintf("%d of its %d resolvers do not answer as the mesh's do", len(failing), len(servers))
	case meshFails > 0 && publicFails > 0:
		reason = "neither mesh names nor public names resolve"
	case meshFails > 0:
		reason = "mesh names do not resolve"
	default:
		reason = "public names do not resolve"
	}
	return Finding{Reason: reason, Said: fmt.Sprintf("within %s: %s", bound, strings.Join(said, " | ")), Toward: failing}
}

// lookTunnel reads the mesh's interface: on a machine reaching the hub, the hub's handshake; on the hub,
// whether any machine has handshaken with it.
func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
	if j.m.Run == nil {
		return Finding{Skip: true}
	}
	hs, err := j.m.Run(ctx, "wg", "show", Interface, "latest-handshakes")
	if err != nil {
		if strings.Contains(err.Error(), "executable file not found") {
			return Finding{Skip: true}
		}
		return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()),
			Toward: []string{TowardHub}}
	}
	ips, err := j.m.Run(ctx, "wg", "show", Interface, "allowed-ips")
	if err != nil {
		return Finding{Reason: "the mesh's tunnel cannot be read or is not up", Said: firstLine(err.Error()),
			Toward: []string{TowardHub}}
	}
	handshakes := map[string]int64{}
	for _, line := range strings.Split(hs, "\n") {
		f := strings.Fields(line)
		if len(f) == 2 {
			at, _ := strconv.ParseInt(f[1], 10, 64)
			handshakes[f[0]] = at
		}
	}
	hub := ""
	for _, line := range strings.Split(ips, "\n") {
		f := strings.Fields(line)
		for _, prefix := range f[min(1, len(f)):] {
			if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
				hub = f[0]
			}
		}
	}
	age := func(at int64) string {
		if at == 0 {
			return "never"
		}
		return now.Sub(time.Unix(at, 0)).Round(time.Second).String() + " ago"
	}
	if hub != "" {
		at := handshakes[hub]
		if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
			return Finding{OK: true}
		}
		return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
			Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
	}
	if len(handshakes) == 0 {
		return Finding{OK: true}
	}
	var newest int64
	for _, at := range handshakes {
		newest = max(newest, at)
	}
	if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
		return Finding{OK: true}
	}
	return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
		Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
}

func (j *Judge) lookBus() Finding {
	if j.m.Linked == nil {
		return Finding{Skip: true}
	}
	if j.m.Linked() {
		return Finding{OK: true}
	}
	return Finding{Reason: "the bus cannot be reached", Said: "no link to the bus is open", Toward: []string{TowardHub}}
}

func (j *Judge) lookRoute() Finding {
	var routes []string
	for _, table := range []struct {
		file          string
		dest, mask, i int
	}{{"route", 1, 7, 0}, {"ipv6_route", 0, 1, 9}} {
		f, err := os.Open(filepath.Join(j.m.ProcNet, table.file))
		if err != nil {
			continue
		}
		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			fields := strings.Fields(scanner.Text())
			if len(fields) <= max(table.dest, table.mask, table.i) || fields[0] == "Iface" {
				continue
			}
			if zero(fields[table.dest]) && zero(fields[table.mask]) && fields[table.i] != "lo" {
				routes = append(routes, fields[table.i])
			}
		}
		f.Close()
	}
	if len(routes) > 0 {
		return Finding{OK: true}
	}
	return Finding{Reason: "the machine has no default route", Said: "no default route in " + j.m.ProcNet}
}

// nameservers is every resolver a file lists, in order.
func nameservers(content string) []string {
	var out []string
	for _, line := range strings.Split(content, "\n") {
		f := strings.Fields(line)
		if len(f) >= 2 && f[0] == "nameserver" {
			out = append(out, f[1])
		}
	}
	return out
}

// waitOf is how long the file tells the C library to wait for one resolver: `options timeout:n`, five
// seconds when it says nothing, and never under one.
func waitOf(content string) time.Duration {
	wait := 5 * time.Second
	for _, line := range strings.Split(content, "\n") {
		f := strings.Fields(line)
		if len(f) == 0 || f[0] != "options" {
			continue
		}
		for _, o := range f[1:] {
			if v, ok := strings.CutPrefix(o, "timeout:"); ok {
				if n, err := strconv.Atoi(v); err == nil {
					wait = time.Duration(max(n, 1)) * time.Second
				}
			}
		}
	}
	return wait
}

func rcodeWords(rcode int) string {
	switch rcode {
	case RcodeNXDomain:
		return "no such name"
	case RcodeServFail:
		return "it failed"
	case RcodeRefused:
		return "it refuses"
	}
	return fmt.Sprintf("code %d", rcode)
}

func typeWords(t uint16) string {
	if t == TypeAAAA {
		return "(IPv6)"
	}
	return "(IPv4)"
}

func listOrNone(s []string) string {
	if len(s) == 0 {
		return "no resolver"
	}
	return strings.Join(s, ", ")
}

func zero(hex string) bool { return strings.Trim(hex, "0") == "" }

func firstLine(s string) string {
	line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
	return line
}

// running is the names of the programs running, from /proc.
func running() []string {
	entries, err := os.ReadDir("/proc")
	if err != nil {
		return nil
	}
	seen := map[string]bool{}
	for _, e := range entries {
		if _, err := strconv.Atoi(e.Name()); err != nil {
			continue
		}
		comm, err := os.ReadFile(filepath.Join("/proc", e.Name(), "comm"))
		if err == nil {
			seen[strings.TrimSpace(string(comm))] = true
		}
	}
	out := make([]string, 0, len(seen))
	for n := range seen {
		out = append(out, n)
	}
	sort.Strings(out)
	return out
}