Merge pull request 'Show the tunnel's peers as each machine sees them, and say on the bus when they move (hq issue 522)' (#225) from feat/522-tunnel-peers into main
This commit was merged in pull request #225.
This commit is contained in:
@@ -62,7 +62,7 @@ func TestTheToolsAreListedUnprefixed(t *testing.T) {
|
||||
for _, tool := range tools() {
|
||||
names[tool.Name] = true
|
||||
}
|
||||
if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 {
|
||||
if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || len(names) != 4 {
|
||||
t.Fatalf("tools: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +1,24 @@
|
||||
// netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches
|
||||
// and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's
|
||||
// own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named),
|
||||
// and what listens here — and nothing that changes anything.
|
||||
// what listens here, and the private network's peers (hq issue 522) — and nothing that changes anything.
|
||||
// Beside them it watches the tunnel and says on the bus when a peer's endpoint moves or its handshake goes
|
||||
// stale or fresh again.
|
||||
// The module's HTTP check is its TypeScript bundle; the runtime serves both under one module.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
go watchTunnel(func(event string, body any) error { return stdio.Emit(event, body) })
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): netcheck.
|
||||
if err := stdio.Serve("", tools()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
@@ -23,6 +28,18 @@ func main() {
|
||||
|
||||
func tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{
|
||||
Name: "netcheck_tunnel",
|
||||
Description: "The private network's peers as this machine sees them (what wg show reads): each peer's machine where " +
|
||||
"the mesh's names say it, its allowed ips, the endpoint (address:port) it was last heard from, its last handshake " +
|
||||
"and that handshake's age, whether it is stale (older than three minutes, or never), and the bytes received and " +
|
||||
"sent. Never a key. On the hub it says what address and port every machine reaches it from.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
cx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
return readTunnel(cx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "netcheck_listening",
|
||||
Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " +
|
||||
@@ -100,3 +117,7 @@ func tools() []stdio.Tool {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func say(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[netcheck] "+format+"\n", args...)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
package main
|
||||
|
||||
// The private network's peers as this machine sees them (hq issue 522): each peer's endpoint, its last
|
||||
// handshake and what crossed it, read where `wg show` reads them. On 2026-10-11 one tunnel leg lost most of
|
||||
// its packets while the machine's own check called its tunnel healthy, and nothing on the mesh could say
|
||||
// which address and port the hub saw for each peer, which is what tells a home router's port mix-up from
|
||||
// anything else. A key is never said: not the interface's private key, which `wg show dump` prints first,
|
||||
// and not a peer's public key either, which names nothing a person reads.
|
||||
//
|
||||
// A watcher says on the bus what changes: a peer's endpoint moving, and its handshake going stale or
|
||||
// fresh again, so the tunnel's own facts are on the bus beside the network check's verdicts.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TunnelInterface is the private network's interface on every machine (the controller's overlay.Interface).
|
||||
const TunnelInterface = "mesh0"
|
||||
|
||||
// staleAfter is how old a handshake may be before the peer is said to be stale. WireGuard renews a
|
||||
// session's handshake every two minutes while traffic flows, and a peer that keeps itself alive sends at
|
||||
// least every 25 seconds, so past three minutes the leg is not carrying what it should.
|
||||
const staleAfter = 3 * time.Minute
|
||||
|
||||
// Peer is one peer of the tunnel, as this machine sees it.
|
||||
type Peer struct {
|
||||
// Name is the machine at the peer's address, where the mesh's names say it; empty when unknown.
|
||||
Name string `json:"name,omitempty"`
|
||||
AllowedIPs []string `json:"allowedIps"`
|
||||
// Endpoint is the address and port this machine last heard the peer from, or sends to; empty
|
||||
// when it has none (a peer that dials in and has not yet).
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
// Handshake is when the last handshake completed; empty when none ever did.
|
||||
Handshake string `json:"latestHandshake,omitempty"`
|
||||
// HandshakeAge is its age in seconds; -1 when none ever did.
|
||||
HandshakeAge int64 `json:"handshakeAgeSeconds"`
|
||||
Stale bool `json:"stale"`
|
||||
Received int64 `json:"receivedBytes"`
|
||||
Sent int64 `json:"sentBytes"`
|
||||
Keepalive string `json:"keepalive,omitempty"`
|
||||
}
|
||||
|
||||
// Tunnel is what netcheck_tunnel answers.
|
||||
type Tunnel struct {
|
||||
Interface string `json:"interface"`
|
||||
ListenPort int `json:"listenPort,omitempty"`
|
||||
Peers []Peer `json:"peers"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// parseDump reads `wg show <interface> dump`: the interface's line first (private key, public key, listen
|
||||
// port, fwmark), then a line per peer (public key, preshared key, endpoint, allowed ips, latest handshake,
|
||||
// received, sent, keepalive). Keys are read past, never kept.
|
||||
func parseDump(out string, now time.Time) (Tunnel, error) {
|
||||
t := Tunnel{Interface: TunnelInterface, Peers: []Peer{}, At: now.UTC().Format(time.RFC3339)}
|
||||
sc := bufio.NewScanner(strings.NewReader(out))
|
||||
first := true
|
||||
for sc.Scan() {
|
||||
f := strings.Split(strings.TrimRight(sc.Text(), "\r"), "\t")
|
||||
if len(f) == 1 && f[0] == "" {
|
||||
continue
|
||||
}
|
||||
if first {
|
||||
first = false
|
||||
if len(f) != 4 {
|
||||
return t, fmt.Errorf("the interface's line has %d fields, not 4", len(f))
|
||||
}
|
||||
t.ListenPort, _ = strconv.Atoi(f[2])
|
||||
continue
|
||||
}
|
||||
if len(f) != 8 {
|
||||
return t, fmt.Errorf("a peer's line has %d fields, not 8", len(f))
|
||||
}
|
||||
p := Peer{AllowedIPs: []string{}, HandshakeAge: -1}
|
||||
if f[2] != "(none)" {
|
||||
p.Endpoint = f[2]
|
||||
}
|
||||
if f[3] != "(none)" {
|
||||
p.AllowedIPs = strings.Split(f[3], ",")
|
||||
}
|
||||
if at, _ := strconv.ParseInt(f[4], 10, 64); at > 0 {
|
||||
when := time.Unix(at, 0)
|
||||
p.Handshake = when.UTC().Format(time.RFC3339)
|
||||
p.HandshakeAge = int64(now.Sub(when) / time.Second)
|
||||
}
|
||||
p.Stale = p.HandshakeAge < 0 || time.Duration(p.HandshakeAge)*time.Second > staleAfter
|
||||
p.Received, _ = strconv.ParseInt(f[5], 10, 64)
|
||||
p.Sent, _ = strconv.ParseInt(f[6], 10, 64)
|
||||
if f[7] != "off" {
|
||||
p.Keepalive = f[7] + "s"
|
||||
}
|
||||
t.Peers = append(t.Peers, p)
|
||||
}
|
||||
if first {
|
||||
return t, errors.New("wg answered nothing about the interface")
|
||||
}
|
||||
sort.Slice(t.Peers, func(i, j int) bool {
|
||||
return strings.Join(t.Peers[i].AllowedIPs, ",") < strings.Join(t.Peers[j].AllowedIPs, ",")
|
||||
})
|
||||
return t, sc.Err()
|
||||
}
|
||||
|
||||
// errNoTunnel is a machine without the interface: not a fault, a machine off the private network.
|
||||
var errNoTunnel = errors.New("this machine has no " + TunnelInterface)
|
||||
|
||||
// names remembers the mesh's name of each peer address once found, so the watcher asks no resolver every look.
|
||||
var names sync.Map
|
||||
|
||||
// readTunnel asks wg for the interface's state and names each peer by the mesh's names.
|
||||
func readTunnel(ctx context.Context) (Tunnel, error) {
|
||||
out, err := exec.CommandContext(ctx, "wg", "show", TunnelInterface, "dump").Output()
|
||||
if err != nil {
|
||||
var ee *exec.ExitError
|
||||
if errors.As(err, &ee) && strings.Contains(string(ee.Stderr), "No such device") {
|
||||
return Tunnel{}, errNoTunnel
|
||||
}
|
||||
if errors.As(err, &ee) && len(ee.Stderr) > 0 {
|
||||
return Tunnel{}, fmt.Errorf("wg show %s: %s", TunnelInterface, strings.TrimSpace(string(ee.Stderr)))
|
||||
}
|
||||
return Tunnel{}, fmt.Errorf("wg show %s: %w", TunnelInterface, err)
|
||||
}
|
||||
t, err := parseDump(string(out), time.Now())
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
for i := range t.Peers {
|
||||
k := peerKey(t.Peers[i])
|
||||
if known, ok := names.Load(k); ok {
|
||||
t.Peers[i].Name = known.(string)
|
||||
continue
|
||||
}
|
||||
if name := nameOf(ctx, t.Peers[i].AllowedIPs); name != "" {
|
||||
names.Store(k, name)
|
||||
t.Peers[i].Name = name
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// nameOf is the mesh's name for a peer's own address (a single-address allowed ip), or "".
|
||||
func nameOf(ctx context.Context, allowed []string) string {
|
||||
for _, a := range allowed {
|
||||
ip, n, err := net.ParseCIDR(a)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if ones, bits := n.Mask.Size(); ones != bits {
|
||||
continue
|
||||
}
|
||||
cx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
names, err := net.DefaultResolver.LookupAddr(cx, ip.String())
|
||||
cancel()
|
||||
if err == nil && len(names) > 0 {
|
||||
name := strings.TrimSuffix(names[0], ".")
|
||||
if host, _, ok := strings.Cut(name, "."); ok {
|
||||
return host
|
||||
}
|
||||
return name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type emitter func(event string, body any) error
|
||||
|
||||
// tunnelWatch remembers each peer's endpoint and staleness at the last look, by its allowed ips.
|
||||
type tunnelWatch struct {
|
||||
primed bool
|
||||
endpoint map[string]string
|
||||
stale map[string]bool
|
||||
}
|
||||
|
||||
func peerKey(p Peer) string { return strings.Join(p.AllowedIPs, ",") }
|
||||
|
||||
func peerBody(p Peer) map[string]any {
|
||||
body := map[string]any{"allowedIps": p.AllowedIPs, "endpoint": p.Endpoint, "handshakeAgeSeconds": p.HandshakeAge}
|
||||
if p.Name != "" {
|
||||
body["peer"] = p.Name
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
// look says what changed since the last look; the first look is silent.
|
||||
func (w *tunnelWatch) look(t Tunnel, emit emitter) {
|
||||
endpoint, stale := map[string]string{}, map[string]bool{}
|
||||
for _, p := range t.Peers {
|
||||
k := peerKey(p)
|
||||
endpoint[k], stale[k] = p.Endpoint, p.Stale
|
||||
if !w.primed {
|
||||
continue
|
||||
}
|
||||
if before, known := w.endpoint[k]; known && before != p.Endpoint && p.Endpoint != "" {
|
||||
body := peerBody(p)
|
||||
body["from"] = before
|
||||
report(emit("tunnel.endpoint-changed", body))
|
||||
}
|
||||
if before, known := w.stale[k]; known && before != p.Stale {
|
||||
if p.Stale {
|
||||
report(emit("tunnel.handshake-stale", peerBody(p)))
|
||||
} else {
|
||||
report(emit("tunnel.handshake-fresh", peerBody(p)))
|
||||
}
|
||||
}
|
||||
}
|
||||
w.endpoint, w.stale, w.primed = endpoint, stale, true
|
||||
}
|
||||
|
||||
func report(err error) {
|
||||
if err != nil {
|
||||
say("an event was not sent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// watchTunnel looks every 20 seconds while this machine has the interface; a machine without it is
|
||||
// looked at again every ten minutes. A failed look is said once and tried again at the next look.
|
||||
func watchTunnel(emit emitter) {
|
||||
var w tunnelWatch
|
||||
said := ""
|
||||
for {
|
||||
cx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
t, err := readTunnel(cx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if err.Error() != said {
|
||||
say("not watching the tunnel: %v", err)
|
||||
said = err.Error()
|
||||
}
|
||||
if errors.Is(err, errNoTunnel) {
|
||||
time.Sleep(10 * time.Minute)
|
||||
} else {
|
||||
time.Sleep(20 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
said = ""
|
||||
w.look(t, emit)
|
||||
time.Sleep(20 * time.Second)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const privateKey = "aPRIVATEkeyThatMustNeverBeSaidAAAAAAAAAAAA="
|
||||
|
||||
func dump(now time.Time) string {
|
||||
fresh := now.Add(-40 * time.Second).Unix()
|
||||
old := now.Add(-10 * time.Minute).Unix()
|
||||
return privateKey + "\tPUBkeyOfTheHubAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t51820\toff\n" +
|
||||
"PUBkeyAceAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:40122\t10.10.0.2/32\t" + itoa(old) + "\t1000\t2000\t25\n" +
|
||||
"PUBkeyShanksAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:51820\t10.10.0.3/32\t" + itoa(fresh) + "\t3000\t4000\toff\n" +
|
||||
"PUBkeyNewAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t(none)\t10.10.0.9/32\t0\t0\t0\toff\n"
|
||||
}
|
||||
|
||||
func itoa(n int64) string { b, _ := json.Marshal(n); return string(b) }
|
||||
|
||||
func TestTheDumpIsReadAndNoKeyIsSaid(t *testing.T) {
|
||||
now := time.Date(2026, 10, 11, 22, 0, 0, 0, time.UTC)
|
||||
tun, err := parseDump(dump(now), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(tun)
|
||||
if strings.Contains(string(raw), "PRIVATE") || strings.Contains(string(raw), "PUBkey") {
|
||||
t.Fatalf("a key is said: %s", raw)
|
||||
}
|
||||
if tun.ListenPort != 51820 || len(tun.Peers) != 3 {
|
||||
t.Fatalf("got %+v", tun)
|
||||
}
|
||||
ace, shanks, fresh := tun.Peers[0], tun.Peers[1], tun.Peers[2]
|
||||
if ace.Endpoint != "84.193.151.6:40122" || ace.HandshakeAge != 600 || !ace.Stale || ace.Keepalive != "25s" || ace.Sent != 2000 {
|
||||
t.Errorf("ace %+v", ace)
|
||||
}
|
||||
if shanks.Stale || shanks.HandshakeAge != 40 || shanks.Keepalive != "" {
|
||||
t.Errorf("shanks %+v", shanks)
|
||||
}
|
||||
if fresh.Endpoint != "" || fresh.HandshakeAge != -1 || !fresh.Stale || fresh.Handshake != "" {
|
||||
t.Errorf("a peer never heard from %+v", fresh)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADumpOfAnotherShapeIsRefused(t *testing.T) {
|
||||
if _, err := parseDump("", time.Now()); err == nil {
|
||||
t.Error("an empty answer was taken")
|
||||
}
|
||||
if _, err := parseDump("a\tb\t1\toff\nshort\tline\n", time.Now()); err == nil {
|
||||
t.Error("a short peer line was taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointsAndHandshakesAreSaidWhenTheyChange(t *testing.T) {
|
||||
var said []string
|
||||
emit := func(event string, body any) error {
|
||||
b := body.(map[string]any)
|
||||
said = append(said, event+":"+b["endpoint"].(string)+":"+asString(b["from"]))
|
||||
return nil
|
||||
}
|
||||
p := func(endpoint string, stale bool) Tunnel {
|
||||
return Tunnel{Peers: []Peer{{AllowedIPs: []string{"10.10.0.2/32"}, Endpoint: endpoint, Stale: stale}}}
|
||||
}
|
||||
var w tunnelWatch
|
||||
w.look(p("1.2.3.4:1", true), emit) // silent: the first look
|
||||
w.look(p("1.2.3.4:1", true), emit)
|
||||
w.look(p("1.2.3.4:2", false), emit)
|
||||
w.look(p("1.2.3.4:2", true), emit)
|
||||
want := "tunnel.endpoint-changed:1.2.3.4:2:1.2.3.4:1,tunnel.handshake-fresh:1.2.3.4:2:,tunnel.handshake-stale:1.2.3.4:2:"
|
||||
if strings.Join(said, ",") != want {
|
||||
t.Errorf("said %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func asString(v any) string {
|
||||
s, _ := v.(string)
|
||||
return s
|
||||
}
|
||||
@@ -1,7 +1,13 @@
|
||||
{
|
||||
"module": "netcheck",
|
||||
"version": "1",
|
||||
"emits": [
|
||||
"tunnel.endpoint-changed",
|
||||
"tunnel.handshake-stale",
|
||||
"tunnel.handshake-fresh"
|
||||
],
|
||||
"tools": [
|
||||
"netcheck_tunnel",
|
||||
"netcheck_tcp",
|
||||
"netcheck_dns",
|
||||
"netcheck_http",
|
||||
@@ -16,6 +22,10 @@
|
||||
"netcheck_dns": [
|
||||
"dig",
|
||||
"nslookup"
|
||||
],
|
||||
"netcheck_tunnel": [
|
||||
"wg show",
|
||||
"wg show mesh0 dump"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
|
||||
Reference in New Issue
Block a user