jschoubben
b1280f81a2
mesh/merge-gate pass: builds netcheck → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered
On novox the tool runner runs as the operator's account, and wg show answered 'Operation not permitted'. Root is the module's concern: the command runs as given when the process is root, else through sudo -n, which the sudo module allows the runtime's account, as nftables does.
265 lines
8.9 KiB
Go
265 lines
8.9 KiB
Go
package main
|
|
|
|
// The private network's peers as this machine sees them (hq issue 522): each peer's endpoint, its last
|
|
// handshake and what crossed it, read where `wg show` reads them. On 2026-10-11 one tunnel leg lost most of
|
|
// its packets while the machine's own check called its tunnel healthy, and nothing on the mesh could say
|
|
// which address and port the hub saw for each peer, which is what tells a home router's port mix-up from
|
|
// anything else. A key is never said: not the interface's private key, which `wg show dump` prints first,
|
|
// and not a peer's public key either, which names nothing a person reads.
|
|
//
|
|
// A watcher says on the bus what changes: a peer's endpoint moving, and its handshake going stale or
|
|
// fresh again, so the tunnel's own facts are on the bus beside the network check's verdicts.
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"os/exec"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// TunnelInterface is the private network's interface on every machine (the controller's overlay.Interface).
|
|
const TunnelInterface = "mesh0"
|
|
|
|
// staleAfter is how old a handshake may be before the peer is said to be stale. WireGuard renews a
|
|
// session's handshake every two minutes while traffic flows, and a peer that keeps itself alive sends at
|
|
// least every 25 seconds, so past three minutes the leg is not carrying what it should.
|
|
const staleAfter = 3 * time.Minute
|
|
|
|
// Peer is one peer of the tunnel, as this machine sees it.
|
|
type Peer struct {
|
|
// Name is the machine at the peer's address, where the mesh's names say it; empty when unknown.
|
|
Name string `json:"name,omitempty"`
|
|
AllowedIPs []string `json:"allowedIps"`
|
|
// Endpoint is the address and port this machine last heard the peer from, or sends to; empty
|
|
// when it has none (a peer that dials in and has not yet).
|
|
Endpoint string `json:"endpoint,omitempty"`
|
|
// Handshake is when the last handshake completed; empty when none ever did.
|
|
Handshake string `json:"latestHandshake,omitempty"`
|
|
// HandshakeAge is its age in seconds; -1 when none ever did.
|
|
HandshakeAge int64 `json:"handshakeAgeSeconds"`
|
|
Stale bool `json:"stale"`
|
|
Received int64 `json:"receivedBytes"`
|
|
Sent int64 `json:"sentBytes"`
|
|
Keepalive string `json:"keepalive,omitempty"`
|
|
}
|
|
|
|
// Tunnel is what netcheck_tunnel answers.
|
|
type Tunnel struct {
|
|
Interface string `json:"interface"`
|
|
ListenPort int `json:"listenPort,omitempty"`
|
|
Peers []Peer `json:"peers"`
|
|
At string `json:"at"`
|
|
}
|
|
|
|
// parseDump reads `wg show <interface> dump`: the interface's line first (private key, public key, listen
|
|
// port, fwmark), then a line per peer (public key, preshared key, endpoint, allowed ips, latest handshake,
|
|
// received, sent, keepalive). Keys are read past, never kept.
|
|
func parseDump(out string, now time.Time) (Tunnel, error) {
|
|
t := Tunnel{Interface: TunnelInterface, Peers: []Peer{}, At: now.UTC().Format(time.RFC3339)}
|
|
sc := bufio.NewScanner(strings.NewReader(out))
|
|
first := true
|
|
for sc.Scan() {
|
|
f := strings.Split(strings.TrimRight(sc.Text(), "\r"), "\t")
|
|
if len(f) == 1 && f[0] == "" {
|
|
continue
|
|
}
|
|
if first {
|
|
first = false
|
|
if len(f) != 4 {
|
|
return t, fmt.Errorf("the interface's line has %d fields, not 4", len(f))
|
|
}
|
|
t.ListenPort, _ = strconv.Atoi(f[2])
|
|
continue
|
|
}
|
|
if len(f) != 8 {
|
|
return t, fmt.Errorf("a peer's line has %d fields, not 8", len(f))
|
|
}
|
|
p := Peer{AllowedIPs: []string{}, HandshakeAge: -1}
|
|
if f[2] != "(none)" {
|
|
p.Endpoint = f[2]
|
|
}
|
|
if f[3] != "(none)" {
|
|
p.AllowedIPs = strings.Split(f[3], ",")
|
|
}
|
|
if at, _ := strconv.ParseInt(f[4], 10, 64); at > 0 {
|
|
when := time.Unix(at, 0)
|
|
p.Handshake = when.UTC().Format(time.RFC3339)
|
|
p.HandshakeAge = int64(now.Sub(when) / time.Second)
|
|
}
|
|
p.Stale = p.HandshakeAge < 0 || time.Duration(p.HandshakeAge)*time.Second > staleAfter
|
|
p.Received, _ = strconv.ParseInt(f[5], 10, 64)
|
|
p.Sent, _ = strconv.ParseInt(f[6], 10, 64)
|
|
if f[7] != "off" {
|
|
p.Keepalive = f[7] + "s"
|
|
}
|
|
t.Peers = append(t.Peers, p)
|
|
}
|
|
if first {
|
|
return t, errors.New("wg answered nothing about the interface")
|
|
}
|
|
sort.Slice(t.Peers, func(i, j int) bool {
|
|
return strings.Join(t.Peers[i].AllowedIPs, ",") < strings.Join(t.Peers[j].AllowedIPs, ",")
|
|
})
|
|
return t, sc.Err()
|
|
}
|
|
|
|
// errNoTunnel is a machine without the interface: not a fault, a machine off the private network.
|
|
var errNoTunnel = errors.New("this machine has no " + TunnelInterface)
|
|
|
|
// names remembers the mesh's name of each peer address once found, so the watcher asks no resolver every look.
|
|
var names sync.Map
|
|
|
|
// readTunnel asks wg for the interface's state and names each peer by the mesh's names.
|
|
func readTunnel(ctx context.Context) (Tunnel, error) {
|
|
program, argv := escalated("wg", []string{"show", TunnelInterface, "dump"}, os.Geteuid())
|
|
out, err := exec.CommandContext(ctx, program, argv...).Output()
|
|
if err != nil {
|
|
var ee *exec.ExitError
|
|
if program == "sudo" && errors.As(err, &ee) && strings.HasPrefix(strings.TrimSpace(string(ee.Stderr)), "sudo:") {
|
|
return Tunnel{}, fmt.Errorf("wg needs root, and sudo refused the runtime's account: %s", strings.TrimSpace(string(ee.Stderr)))
|
|
}
|
|
if errors.As(err, &ee) && strings.Contains(string(ee.Stderr), "No such device") {
|
|
return Tunnel{}, errNoTunnel
|
|
}
|
|
if errors.As(err, &ee) && len(ee.Stderr) > 0 {
|
|
return Tunnel{}, fmt.Errorf("wg show %s: %s", TunnelInterface, strings.TrimSpace(string(ee.Stderr)))
|
|
}
|
|
return Tunnel{}, fmt.Errorf("wg show %s: %w", TunnelInterface, err)
|
|
}
|
|
t, err := parseDump(string(out), time.Now())
|
|
if err != nil {
|
|
return t, err
|
|
}
|
|
for i := range t.Peers {
|
|
k := peerKey(t.Peers[i])
|
|
if known, ok := names.Load(k); ok {
|
|
t.Peers[i].Name = known.(string)
|
|
continue
|
|
}
|
|
if name := nameOf(ctx, t.Peers[i].AllowedIPs); name != "" {
|
|
names.Store(k, name)
|
|
t.Peers[i].Name = name
|
|
}
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
// escalated is the command as it is run: as given when this process is root, else through sudo without a
|
|
// prompt, which the mesh's sudo module allows the runtime's account (ADR 0175 §4: root is the module's
|
|
// concern; the nftables module reads its rules the same way). The tunnel answers only to root.
|
|
func escalated(cmd string, args []string, euid int) (string, []string) {
|
|
if euid == 0 {
|
|
return cmd, args
|
|
}
|
|
return "sudo", append([]string{"-n", cmd}, args...)
|
|
}
|
|
|
|
// nameOf is the mesh's name for a peer's own address (a single-address allowed ip), or "".
|
|
func nameOf(ctx context.Context, allowed []string) string {
|
|
for _, a := range allowed {
|
|
ip, n, err := net.ParseCIDR(a)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if ones, bits := n.Mask.Size(); ones != bits {
|
|
continue
|
|
}
|
|
cx, cancel := context.WithTimeout(ctx, time.Second)
|
|
names, err := net.DefaultResolver.LookupAddr(cx, ip.String())
|
|
cancel()
|
|
if err == nil && len(names) > 0 {
|
|
name := strings.TrimSuffix(names[0], ".")
|
|
if host, _, ok := strings.Cut(name, "."); ok {
|
|
return host
|
|
}
|
|
return name
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type emitter func(event string, body any) error
|
|
|
|
// tunnelWatch remembers each peer's endpoint and staleness at the last look, by its allowed ips.
|
|
type tunnelWatch struct {
|
|
primed bool
|
|
endpoint map[string]string
|
|
stale map[string]bool
|
|
}
|
|
|
|
func peerKey(p Peer) string { return strings.Join(p.AllowedIPs, ",") }
|
|
|
|
func peerBody(p Peer) map[string]any {
|
|
body := map[string]any{"allowedIps": p.AllowedIPs, "endpoint": p.Endpoint, "handshakeAgeSeconds": p.HandshakeAge}
|
|
if p.Name != "" {
|
|
body["peer"] = p.Name
|
|
}
|
|
return body
|
|
}
|
|
|
|
// look says what changed since the last look; the first look is silent.
|
|
func (w *tunnelWatch) look(t Tunnel, emit emitter) {
|
|
endpoint, stale := map[string]string{}, map[string]bool{}
|
|
for _, p := range t.Peers {
|
|
k := peerKey(p)
|
|
endpoint[k], stale[k] = p.Endpoint, p.Stale
|
|
if !w.primed {
|
|
continue
|
|
}
|
|
if before, known := w.endpoint[k]; known && before != p.Endpoint && p.Endpoint != "" {
|
|
body := peerBody(p)
|
|
body["from"] = before
|
|
report(emit("tunnel.endpoint-changed", body))
|
|
}
|
|
if before, known := w.stale[k]; known && before != p.Stale {
|
|
if p.Stale {
|
|
report(emit("tunnel.handshake-stale", peerBody(p)))
|
|
} else {
|
|
report(emit("tunnel.handshake-fresh", peerBody(p)))
|
|
}
|
|
}
|
|
}
|
|
w.endpoint, w.stale, w.primed = endpoint, stale, true
|
|
}
|
|
|
|
func report(err error) {
|
|
if err != nil {
|
|
say("an event was not sent: %v", err)
|
|
}
|
|
}
|
|
|
|
// watchTunnel looks every 20 seconds while this machine has the interface; a machine without it is
|
|
// looked at again every ten minutes. A failed look is said once and tried again at the next look.
|
|
func watchTunnel(emit emitter) {
|
|
var w tunnelWatch
|
|
said := ""
|
|
for {
|
|
cx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
t, err := readTunnel(cx)
|
|
cancel()
|
|
if err != nil {
|
|
if err.Error() != said {
|
|
say("not watching the tunnel: %v", err)
|
|
said = err.Error()
|
|
}
|
|
if errors.Is(err, errNoTunnel) {
|
|
time.Sleep(10 * time.Minute)
|
|
} else {
|
|
time.Sleep(20 * time.Second)
|
|
}
|
|
continue
|
|
}
|
|
said = ""
|
|
w.look(t, emit)
|
|
time.Sleep(20 * time.Second)
|
|
}
|
|
}
|