novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

A working private network, and four reasons it did not work

Browse Source 
Three machines across two sites, two of them behind no reachable address, all
nine paths open. The mesh computes the graph, delivers it as a declaration, and
the nodes bring it up.

Every fault below looked like success from inside the mesh: the graph was
right, the files were right, the services were up, every node reported it had
applied. None was reachable by reasoning.

A running interface does not re-read its configuration. A node joins, every
existing node's peer list changes, the file is replaced -- and the service is
already running, so nothing reloads it. Fixed as declared state rather than a
command: the service must reflect the file. A command to restart would be an
action, and the link may not carry one. The host refused exactly that, which is
how this shape was arrived at.

A hub sharing a site with a spoke appeared twice in that spoke's peer list --
once as a direct peer, once as the route of last resort. WireGuard takes one
entry per key and refuses the file. The ordinary shape of a small mesh, and in
none of the tests written before it ran.

Two nodes at one site that neither can be dialled were peered directly. Nobody
opens the path, and the direct route is more specific than the hub's, so it
wins and blackholes -- this design's own warning arriving in its
implementation. They now route through the hub unless one end can be dialled.

And Docker sets the FORWARD policy to DROP, so a hub with ip_forward enabled
carried nothing between its spokes. The substrate at tier 1 silently breaks the
network at tier 2, and nothing in either tier's state says so. The hub inserts
its own rule above those chains and removes it on the way down.

Two weak tests found by injection along the way: one asserted the keepalive
rule only against the hub, whose peer entries happen not to set that field at
all, so it tested an absence; the other checked the firewall rules by looking
for FORWARD anywhere, which the PostDown line satisfies on its own.
This commit is contained in:
jschoubben
2026-08-29 18:04:15 +02:00
parent f44e73d286
commit 8b974deb42
9 changed files with 504 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-control/main.go
+174
View File
@@ -13,6 +13,7 @@ import (
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
@@ -20,6 +21,7 @@ import (
"github.com/novox/mesh-control/internal/identity"
"github.com/novox/mesh-control/internal/inventory"
"github.com/novox/mesh-control/internal/link"
"github.com/novox/mesh-control/internal/overlay"
"github.com/novox/mesh-control/internal/store"
"github.com/novox/mesh-control/internal/token"
)
@@ -72,6 +74,8 @@ func run() error {
return serve(ctx)
case "declare":
return declare(ctx, args[1:])
case "overlay":
return overlayCommand(ctx, args[1:])
case "version":
fmt.Println(version)
return nil
@@ -96,6 +100,9 @@ func usage() {
broker show where the broker is, and what to expect there
serve consume what nodes say, and answer
declare <node> <file> send a node a signed declaration
overlay place <node> [flags] say where a node is and how it is reached
overlay show the private network, as the mesh computes it
overlay push send every node its part of the private network
version what this binary is
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
@@ -459,3 +466,170 @@ func declare(ctx context.Context, args []string) error {
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
return nil
}
// OverlayCIDRVar is the range the mesh allocates node addresses from.
const OverlayCIDRVar = "MESH_OVERLAY_CIDR"
func overlayCIDR() string {
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
return v
}
return "10.42.0.0/16"
}
func overlayCommand(ctx context.Context, args []string) error {
if len(args) == 0 {
return errors.New("overlay place <node> [flags], overlay show, or overlay push")
}
inv, err := openInventory(ctx)
if err != nil {
return err
}
defer inv.Close()
switch args[0] {
case "place":
return overlayPlace(ctx, inv, args[1:])
case "show":
return overlayShow(ctx, inv)
case "push":
return overlayPush(ctx, inv)
default:
return fmt.Errorf("overlay has no %q; it has place, show and push", args[0])
}
}
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
if len(args) == 0 {
return errors.New("overlay place <node> [--endpoint host:port] [--site name] [--hub]")
}
node := args[0]
set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
site := set.String("site", "", "where this machine physically is, or empty if it roams")
hub := set.Bool("hub", false, "this node is the hub every other routes through")
if err := set.Parse(args[1:]); err != nil {
return err
}
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
// election by address prefix fails silently (novox/hq ADR 0007).
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
return err
}
found, err := inv.NodeByName(ctx, node)
if err != nil {
return err
}
address, err := inv.AssignAddress(ctx, found.ID, overlayCIDR())
if err != nil {
return err
}
fmt.Printf("%s is at %s on the overlay\n", node, address)
switch {
case *hub:
fmt.Println(" the hub — every node not sharing a site routes through it")
case *endpoint == "":
fmt.Println(" not dialable — it opens every path itself")
}
if *site != "" {
fmt.Printf(" at %s, so it peers directly with anything else there\n", *site)
}
return nil
}
// graph reads every node's place and computes the network. Every node at once, which is the whole
// reason this is the control plane's work.
func graph(ctx context.Context, inv *inventory.Inventory) ([]overlay.Node, overlay.Graph, error) {
places, err := inv.Overlays(ctx)
if err != nil {
return nil, nil, err
}
nodes := make([]overlay.Node, 0, len(places))
for _, p := range places {
nodes = append(nodes, overlay.Node{
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
Site: p.Site, Hub: p.Hub, Address: p.Address,
})
}
computed, err := overlay.Compute(nodes, overlayCIDR())
return nodes, computed, err
}
func overlayShow(ctx context.Context, inv *inventory.Inventory) error {
nodes, computed, err := graph(ctx, inv)
if err != nil {
return err
}
if len(nodes) == 0 {
fmt.Println("this mesh has no nodes")
return nil
}
for _, n := range nodes {
place := n.Address
if place == "" {
// Said, not skipped. A node with no place is a node with no network, and it should
// be visible here rather than quietly absent from a list of who is on it.
place = "no address — run `overlay place`"
}
fmt.Printf("%-16s %-14s", n.Name, place)
switch {
case n.Hub:
fmt.Print(" hub")
case !n.Reachable():
fmt.Print(" not dialable")
}
if n.Site != "" {
fmt.Printf(" at %s", n.Site)
}
fmt.Println()
for _, p := range computed[n.Name] {
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
}
}
return nil
}
func overlayPush(ctx context.Context, inv *inventory.Inventory) error {
nodes, computed, err := graph(ctx, inv)
if err != nil {
return err
}
ident, err := openIdentity(ctx)
if err != nil {
return err
}
defer ident.Close()
server, err := link.Connect(nil, nil)
if err != nil {
return err
}
defer server.Close()
sent := 0
for _, n := range nodes {
peers, ok := computed[n.Name]
if !ok {
// Skipped, and said. A node with no key or address is not on the network yet, and
// sending it an empty configuration would take down the one it may already have.
fmt.Printf("%s is not on the overlay yet — skipped\n", n.Name)
continue
}
declaration, err := overlay.Declaration(n, peers, "")
if err != nil {
return err
}
if err := link.Declare(ctx, server.Channel(), ident, n.Name, declaration, 15*time.Second); err != nil {
return err
}
fmt.Printf("sent %s its place on the overlay — %d peer(s)\n", n.Name, len(peers))
sent++
}
fmt.Printf("\n%d of %d node(s) told\n", sent, len(nodes))
return nil
}