From 4fc769ed2904d6f14c19d70424191eb62071c304 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 23:50:17 +0200 Subject: [PATCH 1/2] Show the tunnel's peers as each machine sees them, and say on the bus when they move (hq issue 522) ace's tunnel leg lost most packets while its own check called the tunnel healthy, and nothing on the mesh could say which address and port the hub saw for each peer, which is what tells a home router's port mix-up from anything else. netcheck_tunnel answers each peer's endpoint, last handshake and traffic, never a key, and netcheck now says when a peer's endpoint changes or its handshake goes stale or fresh again. --- modules/netcheck/cmd/netcheck/checks_test.go | 2 +- modules/netcheck/cmd/netcheck/main.go | 23 +- modules/netcheck/cmd/netcheck/tunnel.go | 227 +++++++++++++++++++ modules/netcheck/cmd/netcheck/tunnel_test.go | 81 +++++++ modules/netcheck/module.json | 10 + 5 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 modules/netcheck/cmd/netcheck/tunnel.go create mode 100644 modules/netcheck/cmd/netcheck/tunnel_test.go diff --git a/modules/netcheck/cmd/netcheck/checks_test.go b/modules/netcheck/cmd/netcheck/checks_test.go index fb770c77..4912b663 100644 --- a/modules/netcheck/cmd/netcheck/checks_test.go +++ b/modules/netcheck/cmd/netcheck/checks_test.go @@ -62,7 +62,7 @@ func TestTheToolsAreListedUnprefixed(t *testing.T) { for _, tool := range tools() { names[tool.Name] = true } - if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || len(names) != 3 { + if !names["netcheck_tcp"] || !names["netcheck_dns"] || !names["netcheck_listening"] || !names["netcheck_tunnel"] || len(names) != 4 { t.Fatalf("tools: %v", names) } } diff --git a/modules/netcheck/cmd/netcheck/main.go b/modules/netcheck/cmd/netcheck/main.go index 99f14075..67bf9cf8 100644 --- a/modules/netcheck/cmd/netcheck/main.go +++ b/modules/netcheck/cmd/netcheck/main.go @@ -1,19 +1,24 @@ // netcheck's Go tools bundle (novox/hq ADR 0188, ADR 0193): a process the node's runtime launches // and speaks MCP over stdio to, through the Go SDK. It serves the two checks that are the machine's // own sockets and resolver — a TCP connect, a DNS lookup (of the machine's resolver, or of one named), -// and what listens here — and nothing that changes anything. +// what listens here, and the private network's peers (hq issue 522) — and nothing that changes anything. +// Beside them it watches the tunnel and says on the bus when a peer's endpoint moves or its handshake goes +// stale or fresh again. // The module's HTTP check is its TypeScript bundle; the runtime serves both under one module. package main import ( + "context" "fmt" "os" "strings" + "time" stdio "git.novox.be/novox/mesh-sdk/go" ) func main() { + go watchTunnel(func(event string, body any) error { return stdio.Emit(event, body) }) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): netcheck. if err := stdio.Serve("", tools()); err != nil { fmt.Fprintln(os.Stderr, err) @@ -23,6 +28,18 @@ func main() { func tools() []stdio.Tool { return []stdio.Tool{ + { + Name: "netcheck_tunnel", + Description: "The private network's peers as this machine sees them (what wg show reads): each peer's machine where " + + "the mesh's names say it, its allowed ips, the endpoint (address:port) it was last heard from, its last handshake " + + "and that handshake's age, whether it is stale (older than three minutes, or never), and the bytes received and " + + "sent. Never a key. On the hub it says what address and port every machine reaches it from.", + Run: func(map[string]any) (any, error) { + cx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + return readTunnel(cx) + }, + }, { Name: "netcheck_listening", Description: "What listens on this machine: every TCP socket in the listen state and every bound UDP socket, " + @@ -100,3 +117,7 @@ func tools() []stdio.Tool { }, } } + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[netcheck] "+format+"\n", args...) +} diff --git a/modules/netcheck/cmd/netcheck/tunnel.go b/modules/netcheck/cmd/netcheck/tunnel.go new file mode 100644 index 00000000..b486443f --- /dev/null +++ b/modules/netcheck/cmd/netcheck/tunnel.go @@ -0,0 +1,227 @@ +package main + +// The private network's peers as this machine sees them (hq issue 522): each peer's endpoint, its last +// handshake and what crossed it, read where `wg show` reads them. On 2026-10-11 one tunnel leg lost most of +// its packets while the machine's own check called its tunnel healthy, and nothing on the mesh could say +// which address and port the hub saw for each peer, which is what tells a home router's port mix-up from +// anything else. A key is never said: not the interface's private key, which `wg show dump` prints first, +// and not a peer's public key either, which names nothing a person reads. +// +// A watcher says on the bus what changes: a peer's endpoint moving, and its handshake going stale or +// fresh again, so the tunnel's own facts are on the bus beside the network check's verdicts. + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "os/exec" + "sort" + "strconv" + "strings" + "time" +) + +// TunnelInterface is the private network's interface on every machine (the controller's overlay.Interface). +const TunnelInterface = "mesh0" + +// staleAfter is how old a handshake may be before the peer is said to be stale. WireGuard renews a +// session's handshake every two minutes while traffic flows, and a peer that keeps itself alive sends at +// least every 25 seconds, so past three minutes the leg is not carrying what it should. +const staleAfter = 3 * time.Minute + +// Peer is one peer of the tunnel, as this machine sees it. +type Peer struct { + // Name is the machine at the peer's address, where the mesh's names say it; empty when unknown. + Name string `json:"name,omitempty"` + AllowedIPs []string `json:"allowedIps"` + // Endpoint is the address and port this machine last heard the peer from, or sends to; empty + // when it has none (a peer that dials in and has not yet). + Endpoint string `json:"endpoint,omitempty"` + // Handshake is when the last handshake completed; empty when none ever did. + Handshake string `json:"latestHandshake,omitempty"` + // HandshakeAge is its age in seconds; -1 when none ever did. + HandshakeAge int64 `json:"handshakeAgeSeconds"` + Stale bool `json:"stale"` + Received int64 `json:"receivedBytes"` + Sent int64 `json:"sentBytes"` + Keepalive string `json:"keepalive,omitempty"` +} + +// Tunnel is what netcheck_tunnel answers. +type Tunnel struct { + Interface string `json:"interface"` + ListenPort int `json:"listenPort,omitempty"` + Peers []Peer `json:"peers"` + At string `json:"at"` +} + +// parseDump reads `wg show dump`: the interface's line first (private key, public key, listen +// port, fwmark), then a line per peer (public key, preshared key, endpoint, allowed ips, latest handshake, +// received, sent, keepalive). Keys are read past, never kept. +func parseDump(out string, now time.Time) (Tunnel, error) { + t := Tunnel{Interface: TunnelInterface, Peers: []Peer{}, At: now.UTC().Format(time.RFC3339)} + sc := bufio.NewScanner(strings.NewReader(out)) + first := true + for sc.Scan() { + f := strings.Split(strings.TrimRight(sc.Text(), "\r"), "\t") + if len(f) == 1 && f[0] == "" { + continue + } + if first { + first = false + if len(f) != 4 { + return t, fmt.Errorf("the interface's line has %d fields, not 4", len(f)) + } + t.ListenPort, _ = strconv.Atoi(f[2]) + continue + } + if len(f) != 8 { + return t, fmt.Errorf("a peer's line has %d fields, not 8", len(f)) + } + p := Peer{AllowedIPs: []string{}, HandshakeAge: -1} + if f[2] != "(none)" { + p.Endpoint = f[2] + } + if f[3] != "(none)" { + p.AllowedIPs = strings.Split(f[3], ",") + } + if at, _ := strconv.ParseInt(f[4], 10, 64); at > 0 { + when := time.Unix(at, 0) + p.Handshake = when.UTC().Format(time.RFC3339) + p.HandshakeAge = int64(now.Sub(when) / time.Second) + } + p.Stale = p.HandshakeAge < 0 || time.Duration(p.HandshakeAge)*time.Second > staleAfter + p.Received, _ = strconv.ParseInt(f[5], 10, 64) + p.Sent, _ = strconv.ParseInt(f[6], 10, 64) + if f[7] != "off" { + p.Keepalive = f[7] + "s" + } + t.Peers = append(t.Peers, p) + } + if first { + return t, errors.New("wg answered nothing about the interface") + } + sort.Slice(t.Peers, func(i, j int) bool { + return strings.Join(t.Peers[i].AllowedIPs, ",") < strings.Join(t.Peers[j].AllowedIPs, ",") + }) + return t, sc.Err() +} + +// readTunnel asks wg for the interface's state and names each peer by the mesh's names. +func readTunnel(ctx context.Context) (Tunnel, error) { + out, err := exec.CommandContext(ctx, "wg", "show", TunnelInterface, "dump").Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) && len(ee.Stderr) > 0 { + return Tunnel{}, fmt.Errorf("wg show %s: %s", TunnelInterface, strings.TrimSpace(string(ee.Stderr))) + } + return Tunnel{}, fmt.Errorf("wg show %s: %w", TunnelInterface, err) + } + t, err := parseDump(string(out), time.Now()) + if err != nil { + return t, err + } + for i := range t.Peers { + t.Peers[i].Name = nameOf(ctx, t.Peers[i].AllowedIPs) + } + return t, nil +} + +// nameOf is the mesh's name for a peer's own address (a single-address allowed ip), or "". +func nameOf(ctx context.Context, allowed []string) string { + for _, a := range allowed { + ip, n, err := net.ParseCIDR(a) + if err != nil { + continue + } + if ones, bits := n.Mask.Size(); ones != bits { + continue + } + cx, cancel := context.WithTimeout(ctx, time.Second) + names, err := net.DefaultResolver.LookupAddr(cx, ip.String()) + cancel() + if err == nil && len(names) > 0 { + name := strings.TrimSuffix(names[0], ".") + if host, _, ok := strings.Cut(name, "."); ok { + return host + } + return name + } + } + return "" +} + +type emitter func(event string, body any) error + +// tunnelWatch remembers each peer's endpoint and staleness at the last look, by its allowed ips. +type tunnelWatch struct { + primed bool + endpoint map[string]string + stale map[string]bool +} + +func peerKey(p Peer) string { return strings.Join(p.AllowedIPs, ",") } + +func peerBody(p Peer) map[string]any { + body := map[string]any{"allowedIps": p.AllowedIPs, "endpoint": p.Endpoint, "handshakeAgeSeconds": p.HandshakeAge} + if p.Name != "" { + body["peer"] = p.Name + } + return body +} + +// look says what changed since the last look; the first look is silent. +func (w *tunnelWatch) look(t Tunnel, emit emitter) { + endpoint, stale := map[string]string{}, map[string]bool{} + for _, p := range t.Peers { + k := peerKey(p) + endpoint[k], stale[k] = p.Endpoint, p.Stale + if !w.primed { + continue + } + if before, known := w.endpoint[k]; known && before != p.Endpoint && p.Endpoint != "" { + body := peerBody(p) + body["from"] = before + report(emit("tunnel.endpoint-changed", body)) + } + if before, known := w.stale[k]; known && before != p.Stale { + if p.Stale { + report(emit("tunnel.handshake-stale", peerBody(p))) + } else { + report(emit("tunnel.handshake-fresh", peerBody(p))) + } + } + } + w.endpoint, w.stale, w.primed = endpoint, stale, true +} + +func report(err error) { + if err != nil { + say("an event was not sent: %v", err) + } +} + +// watchTunnel looks every 20 seconds while this machine has the interface; a machine without it is +// looked at again every ten minutes, saying so once. +func watchTunnel(emit emitter) { + var w tunnelWatch + said := "" + for { + cx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + t, err := readTunnel(cx) + cancel() + if err != nil { + if err.Error() != said { + say("not watching the tunnel: %v", err) + said = err.Error() + } + time.Sleep(10 * time.Minute) + continue + } + said = "" + w.look(t, emit) + time.Sleep(20 * time.Second) + } +} diff --git a/modules/netcheck/cmd/netcheck/tunnel_test.go b/modules/netcheck/cmd/netcheck/tunnel_test.go new file mode 100644 index 00000000..0bba1f9e --- /dev/null +++ b/modules/netcheck/cmd/netcheck/tunnel_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +const privateKey = "aPRIVATEkeyThatMustNeverBeSaidAAAAAAAAAAAA=" + +func dump(now time.Time) string { + fresh := now.Add(-40 * time.Second).Unix() + old := now.Add(-10 * time.Minute).Unix() + return privateKey + "\tPUBkeyOfTheHubAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t51820\toff\n" + + "PUBkeyAceAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:40122\t10.10.0.2/32\t" + itoa(old) + "\t1000\t2000\t25\n" + + "PUBkeyShanksAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t84.193.151.6:51820\t10.10.0.3/32\t" + itoa(fresh) + "\t3000\t4000\toff\n" + + "PUBkeyNewAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\t(none)\t(none)\t10.10.0.9/32\t0\t0\t0\toff\n" +} + +func itoa(n int64) string { b, _ := json.Marshal(n); return string(b) } + +func TestTheDumpIsReadAndNoKeyIsSaid(t *testing.T) { + now := time.Date(2026, 10, 11, 22, 0, 0, 0, time.UTC) + tun, err := parseDump(dump(now), now) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(tun) + if strings.Contains(string(raw), "PRIVATE") || strings.Contains(string(raw), "PUBkey") { + t.Fatalf("a key is said: %s", raw) + } + if tun.ListenPort != 51820 || len(tun.Peers) != 3 { + t.Fatalf("got %+v", tun) + } + ace, shanks, fresh := tun.Peers[0], tun.Peers[1], tun.Peers[2] + if ace.Endpoint != "84.193.151.6:40122" || ace.HandshakeAge != 600 || !ace.Stale || ace.Keepalive != "25s" || ace.Sent != 2000 { + t.Errorf("ace %+v", ace) + } + if shanks.Stale || shanks.HandshakeAge != 40 || shanks.Keepalive != "" { + t.Errorf("shanks %+v", shanks) + } + if fresh.Endpoint != "" || fresh.HandshakeAge != -1 || !fresh.Stale || fresh.Handshake != "" { + t.Errorf("a peer never heard from %+v", fresh) + } +} + +func TestADumpOfAnotherShapeIsRefused(t *testing.T) { + if _, err := parseDump("", time.Now()); err == nil { + t.Error("an empty answer was taken") + } + if _, err := parseDump("a\tb\t1\toff\nshort\tline\n", time.Now()); err == nil { + t.Error("a short peer line was taken") + } +} + +func TestEndpointsAndHandshakesAreSaidWhenTheyChange(t *testing.T) { + var said []string + emit := func(event string, body any) error { + b := body.(map[string]any) + said = append(said, event+":"+b["endpoint"].(string)+":"+asString(b["from"])) + return nil + } + p := func(endpoint string, stale bool) Tunnel { + return Tunnel{Peers: []Peer{{AllowedIPs: []string{"10.10.0.2/32"}, Endpoint: endpoint, Stale: stale}}} + } + var w tunnelWatch + w.look(p("1.2.3.4:1", true), emit) // silent: the first look + w.look(p("1.2.3.4:1", true), emit) + w.look(p("1.2.3.4:2", false), emit) + w.look(p("1.2.3.4:2", true), emit) + want := "tunnel.endpoint-changed:1.2.3.4:2:1.2.3.4:1,tunnel.handshake-fresh:1.2.3.4:2:,tunnel.handshake-stale:1.2.3.4:2:" + if strings.Join(said, ",") != want { + t.Errorf("said %v", said) + } +} + +func asString(v any) string { + s, _ := v.(string) + return s +} diff --git a/modules/netcheck/module.json b/modules/netcheck/module.json index c8f87901..ec01003a 100644 --- a/modules/netcheck/module.json +++ b/modules/netcheck/module.json @@ -1,7 +1,13 @@ { "module": "netcheck", "version": "1", + "emits": [ + "tunnel.endpoint-changed", + "tunnel.handshake-stale", + "tunnel.handshake-fresh" + ], "tools": [ + "netcheck_tunnel", "netcheck_tcp", "netcheck_dns", "netcheck_http", @@ -16,6 +22,10 @@ "netcheck_dns": [ "dig", "nslookup" + ], + "netcheck_tunnel": [ + "wg show", + "wg show mesh0 dump" ] }, "build": { From c31e29f9a6017ceb815217570b422a015062ab2c Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 23:51:40 +0200 Subject: [PATCH 2/2] Keep watching the tunnel after a failed look, and remember peers' names --- modules/netcheck/cmd/netcheck/tunnel.go | 28 ++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/modules/netcheck/cmd/netcheck/tunnel.go b/modules/netcheck/cmd/netcheck/tunnel.go index b486443f..439b7556 100644 --- a/modules/netcheck/cmd/netcheck/tunnel.go +++ b/modules/netcheck/cmd/netcheck/tunnel.go @@ -20,6 +20,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" ) @@ -109,11 +110,20 @@ func parseDump(out string, now time.Time) (Tunnel, error) { return t, sc.Err() } +// errNoTunnel is a machine without the interface: not a fault, a machine off the private network. +var errNoTunnel = errors.New("this machine has no " + TunnelInterface) + +// names remembers the mesh's name of each peer address once found, so the watcher asks no resolver every look. +var names sync.Map + // readTunnel asks wg for the interface's state and names each peer by the mesh's names. func readTunnel(ctx context.Context) (Tunnel, error) { out, err := exec.CommandContext(ctx, "wg", "show", TunnelInterface, "dump").Output() if err != nil { var ee *exec.ExitError + if errors.As(err, &ee) && strings.Contains(string(ee.Stderr), "No such device") { + return Tunnel{}, errNoTunnel + } if errors.As(err, &ee) && len(ee.Stderr) > 0 { return Tunnel{}, fmt.Errorf("wg show %s: %s", TunnelInterface, strings.TrimSpace(string(ee.Stderr))) } @@ -124,7 +134,15 @@ func readTunnel(ctx context.Context) (Tunnel, error) { return t, err } for i := range t.Peers { - t.Peers[i].Name = nameOf(ctx, t.Peers[i].AllowedIPs) + k := peerKey(t.Peers[i]) + if known, ok := names.Load(k); ok { + t.Peers[i].Name = known.(string) + continue + } + if name := nameOf(ctx, t.Peers[i].AllowedIPs); name != "" { + names.Store(k, name) + t.Peers[i].Name = name + } } return t, nil } @@ -204,7 +222,7 @@ func report(err error) { } // watchTunnel looks every 20 seconds while this machine has the interface; a machine without it is -// looked at again every ten minutes, saying so once. +// looked at again every ten minutes. A failed look is said once and tried again at the next look. func watchTunnel(emit emitter) { var w tunnelWatch said := "" @@ -217,7 +235,11 @@ func watchTunnel(emit emitter) { say("not watching the tunnel: %v", err) said = err.Error() } - time.Sleep(10 * time.Minute) + if errors.Is(err, errNoTunnel) { + time.Sleep(10 * time.Minute) + } else { + time.Sleep(20 * time.Second) + } continue } said = ""