From 7283924a3552a479ac131486b062e2e04507f0c2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 23 Sep 2026 23:26:35 +0200 Subject: [PATCH 1/2] Take over the found tunnel: its key, its port, its peers; stop it, never flush MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On an adopted machine the private network takes the predecessor's tunnel over in place (hq ADR 0105). Genesis finds the one interface up besides the mesh's own, settles the hub's port and the mesh's range on it, and skips ADR 0100's non-overlap check for a range that is now the tunnel's; a --hub-port or --overlay-range that disagrees is refused naming the tunnel's. At enrolment the found interface's private key becomes this node's overlay key — the one credential the mesh takes rather than mints — stored where a generated one is stored, never printed and never sent; the tunnel (port, address, range, peers) travels with the keys so the mesh composes from it before the first declaration. The interface's service may say what it takes over. Before the mesh's unit starts, the found configuration is kept like any held file and the found unit is stopped and disabled; nothing is flushed, and an interface still up after its unit stopped refuses the takeover rather than half-working. The report says what was carried: interface, port, range, peer count, taken or not, and where the original was kept. --- cmd/mesh-bootstrap/main.go | 6 + cmd/mesh-host/main.go | 57 +++++- internal/apply/apply.go | 36 ++++ internal/apply/hold_test.go | 5 + internal/apply/takeover.go | 159 +++++++++++++++ internal/apply/takeover_test.go | 165 ++++++++++++++++ internal/bootstrap/bootstrap.go | 26 +++ internal/bootstrap/enrol.go | 9 +- internal/bootstrap/ports.go | 48 ++++- internal/bootstrap/ports_test.go | 85 +++++++- internal/declaration/adoption_test.go | 25 +++ internal/declaration/declaration.go | 36 ++++ internal/identity/overlay.go | 21 ++ internal/identity/overlay_test.go | 28 +++ internal/link/enrol.go | 28 ++- internal/link/messages.go | 15 ++ internal/tunnel/tunnel.go | 274 ++++++++++++++++++++++++++ internal/tunnel/tunnel_test.go | 172 ++++++++++++++++ 18 files changed, 1182 insertions(+), 13 deletions(-) create mode 100644 internal/apply/takeover.go create mode 100644 internal/apply/takeover_test.go create mode 100644 internal/identity/overlay_test.go create mode 100644 internal/tunnel/tunnel.go create mode 100644 internal/tunnel/tunnel_test.go diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index c001e7a..fb8c777 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -140,6 +140,10 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh firewall stay as they are, the foundation's filter is not loaded and the mesh guards its own ports instead, and each module is taken on it one at a time. Without it, a machine in use is refused + --tunnel adopted: the interface of the tunnel the private network takes over + (its key, port, range and peers); found by itself when one is up, and + needed only when several are. --hub-port and --overlay-range then + follow the tunnel The installer carries a builder, not a control plane. What raises a mesh is therefore the same thing that will maintain it, and the control plane a mesh ends up running is @@ -334,6 +338,8 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { "raise this machine adopted: keep what it runs and its firewall until each module is taken") set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange, "the private network's address range; must not overlap a tunnel the machine already runs") + set.StringVar(&opts.Tunnel, "tunnel", "", + "adopted: the found tunnel's interface the private network takes over; found by itself when one is up") if opts.Answers == nil { opts.Answers = map[string]string{} } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 681eb3c..a8f420a 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -36,6 +36,7 @@ import ( "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" + "github.com/novox/mesh-host/internal/tunnel" "github.com/novox/mesh-host/internal/upgrade" ) @@ -88,6 +89,7 @@ type options struct { state string token string nodeName string + tunnel string dryRun bool file string } @@ -116,6 +118,8 @@ func parseArgs(args []string) (string, options, error) { set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing") set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person") set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries") + set.StringVar(&opts.tunnel, "tunnel", "", "enrol, adopted: the found tunnel's interface whose key "+ + "this node takes as its own; found by itself when one is up") // Parsed in a loop, because the standard library stops at the FIRST non-flag argument. // `mesh-host inventory --json` hit that once, and taking the subcommand off the front @@ -478,14 +482,39 @@ func enrol(ctx context.Context, opts options) error { } fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64()) - // Its key on the private network, generated here and now for the same reason: the private + // Its key on the private network. Generated here and now, for the same reason: the private // half must never have been anywhere else. The mesh receives only the public half and uses it // to compute a graph it cannot impersonate. - mine.Overlay, err = identity.GenerateOverlayKey() - if err != nil { - return err + // + // **Except on an adopted node with a tunnel** (novox/hq ADR 0105): the found interface's key + // becomes this node's, so the peers that know the tunnel by that key keep reaching it once + // the mesh's interface takes the tunnel over. The one case where the mesh takes a credential + // it did not mint — read from the found configuration, written where a generated one is + // written, never printed, never sent. + var found *link.Tunnel + if token.Adopted { + tun, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) + switch { + case errors.Is(err, tunnel.ErrNone): + fmt.Println("no tunnel is up on this machine; the private network's key is generated") + case err != nil: + return err + default: + mine.Overlay, err = identity.OverlayKeyFrom(tun.PrivateKey()) + if err != nil { + return err + } + found = carried(tun) + fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public) + } + } + if found == nil { + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + return err + } + fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) } - fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) // And the key secrets are sealed to. Here, with the others, because the mesh cannot seal // anything to a key it has not been told about — a key made later would leave a node that @@ -519,7 +548,8 @@ func enrol(ctx context.Context, opts options) error { proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public, sealing.Public, serving.Public)) reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret, - mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, opts.timeout) + mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found, + opts.timeout) if err != nil { return err } @@ -578,6 +608,16 @@ func enrol(ctx context.Context, opts options) error { return nil } +// carried is a found tunnel as it is presented to the mesh: everything but its private key. +func carried(t tunnel.Found) *link.Tunnel { + out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, + Address: t.Address, Range: t.Range, PublicKey: t.PublicKey} + for _, p := range t.Peers { + out.Peers = append(out.Peers, link.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address}) + } + return out +} + func firstNonEmpty(values ...string) string { for _, v := range values { if strings.TrimSpace(v) != "" { @@ -891,6 +931,11 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D if updated.Firewall != nil { report.Firewall = updated.Firewall.Kind } + // And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105). + if t := outcome.Tunnel; t != nil { + report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range, + Peers: t.Peers, Taken: t.Taken, Kept: t.Kept} + } reached, err := reachable.Collect(ctx, apply.ExecRunner) if err != nil { fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index d473f3f..c2fd918 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -57,6 +57,9 @@ type Outcome struct { // Report is what an apply did, in the order it did it. type Report struct { Outcomes []Outcome `json:"outcomes"` + // Tunnel is what this apply says about the tunnel the private network took over, when the + // declaration names one (novox/hq ADR 0105). + Tunnel *TakenTunnel `json:"tunnel,omitempty"` } // Changed reports whether anything about the machine actually moved. An apply that changed @@ -153,6 +156,11 @@ func ApplyKeeping( for _, r := range d.Resources { declared[r.Identity()] = true } + if svc := takesOver(d); svc != nil { + // The found tunnel's configuration is held under an id of its own, declared for as long + // as the service that took it over is (novox/hq ADR 0105). + declared[takeOverID(svc)] = true + } // Which firewall is found here, before anything else, since an unsupported one refuses the // whole declaration (novox/hq ADR 0100). Nothing for a converged node. @@ -327,6 +335,29 @@ func ApplyKeeping( } } + // The private network takes over the tunnel it found, ahead of the service that replaces + // it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never + // flushed. A failure here fails the service too — the mesh's interface is not started on a + // port the found one still holds. + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { + var outcome Outcome + var facts TakenTunnel + var err error + if d.Adoption == nil { + err = errNotAdopted + } else { + outcome, facts, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) + } + if err != nil { + failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) + continue + } + report.Outcomes = append(report.Outcomes, outcome) + report.Tunnel = &facts + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } + was, _ := known.Find(resource.Identity()) var outcome Outcome var err error @@ -393,6 +424,11 @@ func ApplyKeeping( known.Release(held.ID) outcome.Detail = takenDetail(held) } + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { + // The found interface is down and the mesh's is up in its place: the tunnel changed + // hands (novox/hq ADR 0105). + report.Tunnel.Taken = true + } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { changed[resource.Identity()] = true diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 0651f4f..3e95e36 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -19,6 +19,8 @@ import ( type machine struct { containers map[string]*fakeContainer asked []string + // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. + wgUp string // units are service units by name, as systemd would report them; volumes are the runtime's // named volumes. @@ -94,6 +96,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e if name == "systemctl" { return m.systemctl(args) } + if name == "wg" { + return m.wgUp, nil + } if name == "getent" { if m.users[args[len(args)-1]] { return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go new file mode 100644 index 0000000..bff4935 --- /dev/null +++ b/internal/apply/takeover.go @@ -0,0 +1,159 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" + "github.com/novox/mesh-host/internal/tunnel" +) + +// The private network takes over the tunnel it found (novox/hq ADR 0105). +// +// The controller says so on the interface's service: `takes-over` names the found interface, the +// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps +// that file like any held file — the original recorded before anything else happens to it — and +// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the +// file is never written, and the found interface goes down the way its own unit takes it down. +// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found +// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands. +// +// Every apply, not once: a found unit somebody starts again would take the port back from the +// mesh's interface, so it is stopped again and said so. That is the one place an adopted node +// undoes something done by hand, and it is because the tunnel is the mesh's now. + +// TakenTunnel is what an apply says about a tunnel it took over, for the node's report. +type TakenTunnel struct { + Interface string + Port int + Range string + Peers int + // Taken is whether the found interface is down and disabled and the mesh's up in its place. + Taken bool + Kept string +} + +// takeOverID is the held record's id for the found configuration: the service's own with a suffix, +// so it is declared for as long as the service is and never mistaken for the service itself. +func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" } + +// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that +// replaces it. Returned is the hold's outcome, and what was found for the report. +func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration, + known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) { + t := svc.TakesOver + id := takeOverID(svc) + module, _ := d.Adoption.UntakenModuleOf(svc.ID) + if module == "" { + module = "the private network" + } + facts := TakenTunnel{Interface: t.Interface} + + // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so + // the same code keeps its original, digests it and notices it changing. + file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} + was, already := known.HeldAt(id) + out, held, err := hold(ctx, sys, file, module, was, already, + "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) + if err != nil { + return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + } + known.RecordHeld(held) + facts.Kept = held.Kept + // What the file says, for the report: read from the machine, or from the kept original when + // the machine's copy is gone. The private key stays in the file; nothing here keeps it. + unread := "" + raw, err := os.ReadFile(t.Config) + if err != nil && held.Kept != "" { + raw, err = os.ReadFile(held.Kept) + } + if err == nil { + if found, perr := tunnel.Parse(raw); perr == nil { + facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) + } else { + unread = perr.Error() + } + } else { + unread = err.Error() + } + + // 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not + // there is not an error — the interface may have been raised another way, which the check + // below catches — and neither is one already down. + var did []string + state, err := sys.ServiceState(ctx, run, t.Unit) + switch { + case err != nil: + did = append(did, t.Unit+" is not a unit here") + case state == "running": + if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil { + return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err) + } + after, err := sys.ServiceState(ctx, run, t.Unit) + if err != nil { + return out, facts, err + } + if after != "stopped" { + return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) + } + did = append(did, "stopped "+t.Unit) + } + if err == nil { + if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" { + if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil { + return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) + } + did = append(did, "disabled it at boot") + } + } + + // 3. The interface is gone. If it is still up, something other than its unit raised it, and + // starting the mesh's on the same port and address would fail or, worse, half work. + if up, err := run(ctx, "wg", "show", "interfaces"); err == nil { + for _, iface := range strings.Fields(up) { + if iface == t.Interface { + return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+ + "than that unit raises it, and the mesh's interface cannot take its port and address "+ + "while it does. Nothing was flushed", t.Interface, t.Unit) + } + } + } + + out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" + if held.Kept != "" { + out.Detail += " (original at " + held.Kept + ")" + } + if len(did) > 0 { + out.Detail += "; " + strings.Join(did, ", ") + " — never flushed" + } + if held.Changed != "" { + out.Detail += "; " + held.Changed + " by something other than the mesh since it was found" + } + if unread != "" { + // Said, not swallowed: the report would otherwise say a tunnel with no port and no + // peers was carried, which reads as a tunnel that was not one. + out.Detail += "; what it says could not be read as a tunnel's: " + unread + } + return out, facts, nil +} + +// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since +// a machine has one private network. +func takesOver(d *declaration.Declaration) *declaration.Service { + for _, r := range d.Resources { + if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil { + return svc + } + } + return nil +} + +// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the +// parser refuses already; kept as a second line of defence at the point of acting. +var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only") diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go new file mode 100644 index 0000000..2ca9abf --- /dev/null +++ b/internal/apply/takeover_test.go @@ -0,0 +1,165 @@ +package apply + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the +// found interface without flushing it, and keeps its configuration. + +// foundConf is the predecessor's configuration, with a real key made once per run: the key is +// what the takeover must never print or copy, so it had better be one. +var foundConf = func() string { + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + return "[Interface]\nPrivateKey = " + base64.StdEncoding.EncodeToString(k.Bytes()) + "\n" + + "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + + "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" +}() + +// aTakeover is the private network's declaration for an adopted hub whose interface takes over +// the found tunnel: the mesh's configuration — with the found key set from the node's own key file +// and the found peers in its list — and the interface's service naming what it replaces. +func aTakeover(t *testing.T, config, mesh string) *declaration.Declaration { + t.Helper() + return adopted(t, + `{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`, + `{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600", + "content":"[Interface]\nAddress = 192.0.2.1/32\nListenPort = 51900\nPostUp = wg set %i private-key /var/lib/mesh-host/overlay.key\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, + {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled", + "restart-on":["mesh-wireguard.overlay-config"], + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`) +} + +// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet. +func aHubInUse(t *testing.T) (dir, config, mesh string, m *machine) { + t.Helper() + dir = t.TempDir() + config = filepath.Join(dir, "wg0.conf") + mesh = filepath.Join(dir, "mesh0.conf") + if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil { + t.Fatal(err) + } + m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{ + "wg-quick@wg0": {active: "active", enabled: "enabled"}, + "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, + }} + return dir, config, mesh, m +} + +func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) { + dir, config, mesh, m := aHubInUse(t) + report, state := applyAdopted(t, aTakeover(t, config, mesh), store.State{}, m, dir) + + // The found interface: its unit stopped and disabled, and nothing else done to it. + if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { + t.Fatalf("the found unit was not stopped and disabled: %+v", u) + } + for _, asked := range m.asked { + if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") { + t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked) + } + if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") { + t.Errorf("the found interface was flushed: %q", asked) + } + } + // Its configuration: on disk as it was, its original kept, held for the module. + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatalf("the found configuration was changed:\n%s", got) + } + held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over") + if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" { + t.Fatalf("the found configuration is not held: %+v", held) + } + if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf { + t.Fatalf("the original was not kept as found: %q", kept) + } + // The mesh's interface: up, enabled, with the found peers in the file the mesh wrote. + if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" { + t.Fatalf("the mesh's interface was not raised: %+v", u) + } + if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") { + t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got) + } + // And the report says so, with what was found — port, range, peers — and never the key. + if report.Tunnel == nil || !report.Tunnel.Taken || report.Tunnel.Port != 51900 || + report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept { + t.Fatalf("the report does not say what was carried: %+v", report.Tunnel) + } + private := strings.TrimSpace(strings.SplitN(strings.SplitN(foundConf, "PrivateKey = ", 2)[1], "\n", 2)[0]) + for _, o := range report.Outcomes { + if strings.Contains(o.Detail, private) { + t.Errorf("the found key was printed in an outcome: %+v", o) + } + } + if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" || + !strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") { + t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o) + } + if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded { + t.Error("the found configuration was recorded as applied, so it would be removed as an orphan") + } +} + +func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) { + dir, config, mesh, m := aHubInUse(t) + d := aTakeover(t, config, mesh) + _, state := applyAdopted(t, d, store.State{}, m, dir) + m.asked = nil + + report, again := applyAdopted(t, d, state, m, dir) + if report.Changed() { + t.Errorf("a second apply moved the machine: %+v", report.Outcomes) + } + if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still { + t.Error("the hold on the found configuration was forgotten while the service still declares it") + } + if m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit already down was stopped again") + } + + // Somebody starts the found unit again: it would take the port back, so it is stopped again + // — the one thing on an adopted node the mesh undoes, because the tunnel is the mesh's now. + m.units["wg-quick@wg0"].active = "active" + m.asked = nil + _, _ = applyAdopted(t, d, again, m, dir) + if m.units["wg-quick@wg0"].active != "inactive" || !m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit started again was left holding the mesh's port") + } +} + +func TestAFoundInterfaceStillUpAfterItsUnitStoppedRefusesTheTakeover(t *testing.T) { + dir, config, mesh, m := aHubInUse(t) + m.wgUp = "wg0 mesh0\n" + _, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh), store.State{}, + store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "still up") || !strings.Contains(err.Error(), "Nothing was flushed") { + t.Fatalf("an interface something else raises was taken over anyway: %v", err) + } + if m.units["wg-quick@mesh0"].active == "active" { + t.Error("the mesh's interface was started on a port the found one still holds") + } + if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { + t.Error("the found configuration was not kept before the refusal") + } +} + +func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) { + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`)) + if err == nil || !strings.Contains(err.Error(), "adopted") { + t.Fatalf("a takeover on a converged node was accepted: %v", err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 4349bcc..12089b0 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -36,6 +36,7 @@ import ( "time" "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/tunnel" ) // Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence @@ -201,6 +202,11 @@ type Options struct { // until each module is taken, its firewall stays in force, and the mesh guards its own ports // in a table that only refuses. Without it, a machine in use is refused. Adopted bool + // Tunnel names the found tunnel's interface an adopted hub takes over (novox/hq ADR 0105), when + // more than one is up and the machine cannot say which. Empty finds the one that is up. Once + // found, the tunnel's port is the hub's and its range the private network's; --hub-port and + // --overlay-range may agree with it or be left unsaid. + Tunnel string } // pivots reports whether this run goes past the foundation. @@ -311,6 +317,9 @@ type Result struct { // Filter is the packet filter chosen for when the node converges; an adopted genesis loads // none, and the flip assigns this one. Filter string `json:"filter-on-converge,omitempty"` + // Tunnel is the found tunnel an adopted genesis takes over (novox/hq ADR 0105): what was read + // from it, never its key. + Tunnel *tunnel.Found `json:"tunnel,omitempty"` } // Run performs the bootstrap, saying what it is doing as it goes. @@ -394,6 +403,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } result.Firewall = string(kind) say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force") + + // The tunnel the predecessor left, which the private network takes over (novox/hq ADR + // 0105): its port is the hub's and its range is the mesh's from here on, so both are + // settled before the ports are checked free and the bundle rewritten. + found, err := TakeTheTunnel(&o, d.Run) + if err != nil { + return result, failed(StepPreflight, err) + } + if found != nil { + result.Tunnel = found + result.Ports = o.Ports + say(fmt.Sprintf(" tunnel %s — the private network takes it over: its port %d is "+ + "the hub's, its range %s the mesh's, and its %d peer(s) are carried until they enrol", + found.Interface, found.Port, found.Range, len(found.Peers))) + } else { + say(" tunnel none up on this machine; the private network is raised on its own port and range") + } } sys, err := WorkOutSystem(ctx, d.Run, o.System) diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index 7ee3b71..0137c3b 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -112,7 +112,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla return out, err } joining, cancel := context.WithTimeout(ctx, o.Wait) - joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) + args := []string{"enrol", "--token", token, "--state", o.State} + if o.Tunnel != "" { + // The found tunnel's key becomes this node's overlay key, and the tunnel travels with + // the enrolment (novox/hq ADR 0105). Named, so the host takes the one genesis settled + // its ports and range on and not another that came up since. + args = append(args, "--tunnel", o.Tunnel) + } + joined, err := control.run(joining, o.Host, args...) cancel() if err != nil { return out, fmt.Errorf( diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go index c3c1ed8..cb09372 100644 --- a/internal/bootstrap/ports.go +++ b/internal/bootstrap/ports.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net" "sort" @@ -13,6 +14,7 @@ import ( "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/tunnel" ) // FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100). @@ -321,6 +323,36 @@ func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(rea return nil } +// TakeTheTunnel finds the tunnel an adopted machine's private network takes over (novox/hq ADR +// 0105) and settles the options on it: the hub's port is the tunnel's, the mesh's range is the +// tunnel's, and the interface is named for the enrolment that takes its key. Nil when no tunnel is +// up, which is an ordinary machine. A --hub-port or --overlay-range that disagrees with the +// tunnel is refused: the peers dial the tunnel's port and live in its range, and a mesh raised +// beside them on other numbers is the two-tunnel shape the record rejects. +func TakeTheTunnel(o *Options, run Runner) (*tunnel.Found, error) { + found, err := tunnel.Find(context.Background(), tunnel.Runner(run), o.Tunnel) + if errors.Is(err, tunnel.ErrNone) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("%w. An adopted hub takes over the tunnel it finds; nothing was changed", err) + } + if o.Ports.Hub != 0 && o.Ports.Hub != DefaultPorts().Hub && o.Ports.Hub != found.Port { + return nil, fmt.Errorf("--hub-port %d disagrees with the tunnel %s, which listens on %d: the "+ + "private network takes over that tunnel on its own port, so leave --hub-port unsaid or "+ + "say %d", o.Ports.Hub, found.Interface, found.Port, found.Port) + } + if o.OverlayRange != "" && o.OverlayRange != DefaultOverlayRange && o.OverlayRange != found.Range { + return nil, fmt.Errorf("--overlay-range %s disagrees with the tunnel %s, whose range is %s: the "+ + "private network takes over that tunnel with its range, so leave --overlay-range unsaid "+ + "or say %s", o.OverlayRange, found.Interface, found.Range, found.Range) + } + o.Tunnel = found.Interface + o.Ports.Hub = found.Port + o.OverlayRange = found.Range + return &found, nil +} + // OverlayClear refuses a private-network range that overlaps an address or a route the machine // already has — a predecessor's tunnel still running — naming the interface. The mesh's own // interface is not counted. @@ -459,12 +491,26 @@ func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declara } return false } + if o.Tunnel != "" { + // The hub's port is the found tunnel's, held by that tunnel until the mesh's interface + // takes it over (novox/hq ADR 0105): held by design, not by something else. + inner := ours + ours = func(r reachable.Reach) bool { + return inner(r) || (r.Protocol == "udp" && r.Port == p.Hub) + } + } if err := PortsFree(ctx, run, p, ours); err != nil { return err } say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp", p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub)) - if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { + if o.Tunnel != "" { + // One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the + // two must not overlap applies only where a found tunnel is left running beside the mesh's + // (ADR 0100, narrowed by ADR 0105). + say(fmt.Sprintf(" range %s is the tunnel %s's, taken over; not checked against it", + o.OverlayRange, o.Tunnel)) + } else if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { return err } if err := NamesFree(ctx, run, names, known); err != nil { diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index 366a3dd..ac0ec92 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -2,6 +2,9 @@ package bootstrap import ( "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" "errors" "os" "path/filepath" @@ -12,6 +15,7 @@ import ( "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/tunnel" ) // Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free, @@ -130,9 +134,9 @@ func TestTwoThingsOnOnePortAreRefused(t *testing.T) { // machineRunner answers ss, docker ps, docker inspect and ip from fixtures. type machineRunner struct { - ss, ps, addrs, routes string - unlabelled map[string]bool - labelled map[string]bool + ss, ps, addrs, routes, wg string + unlabelled map[string]bool + labelled map[string]bool } func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) { @@ -154,6 +158,8 @@ func (m machineRunner) run(_ context.Context, name string, args ...string) (stri return m.addrs, nil case name == "ip" && args[1] == "route": return m.routes, nil + case name == "wg": + return m.wg, nil } return "", nil } @@ -295,3 +301,76 @@ func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) { t.Error("a container under the package registry's name on a fresh machine was not refused") } } + +// novox/hq ADR 0105: an adopted genesis takes over the tunnel it finds — its port is the hub's, +// its range the mesh's, and neither is refused for being held by it. +func TestAnAdoptedGenesisSettlesOnTheTunnelItFinds(t *testing.T) { + private, err := aFoundKey() + if err != nil { + t.Fatal(err) + } + conf := "[Interface]\nPrivateKey = " + private + "\nListenPort = 51900\nAddress = 192.0.2.1/24\n" + + "[Peer]\nPublicKey = PEER=\nAllowedIPs = 192.0.2.2/32\n" + tunnel.ReadFile = func(path string) ([]byte, error) { + if path == tunnel.ConfigDir+"/wg0.conf" { + return []byte(conf), nil + } + return nil, errors.New("no such file") + } + t.Cleanup(func() { tunnel.ReadFile = os.ReadFile }) + m := machineRunner{ + wg: "wg0\n", + ss: "udp UNCONN 0 0 0.0.0.0:51900 0.0.0.0:*\n", + addrs: "5: wg0 inet 192.0.2.1/24 scope global wg0\n", + routes: "192.0.2.0/24 dev wg0 proto kernel scope link src 192.0.2.1\n", + } + + o := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange, State: filepath.Join(t.TempDir(), "state.json")} + found, err := TakeTheTunnel(&o, m.run) + if err != nil || found == nil { + t.Fatalf("the tunnel was not found and taken: %+v %v", found, err) + } + if o.Tunnel != "wg0" || o.Ports.Hub != 51900 || o.OverlayRange != "192.0.2.0/24" { + t.Fatalf("genesis did not settle on the tunnel's port and range: %+v", o) + } + // Its port is held by the tunnel and its range overlaps the tunnel's — by design, not refused. + if err := CheckTheMachine(context.Background(), o, m.run, producedBundle(t).Declaration, func(string) {}); err != nil { + t.Fatalf("the machine was refused for the tunnel it takes over: %v", err) + } + // Whereas the same machine not taking it over is refused on both counts (ADR 0100). + plain := o + plain.Tunnel = "" + if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil || + !strings.Contains(err.Error(), "51900") { + t.Fatalf("a tunnel not taken over stopped being refused for holding the hub's port: %v", err) + } + plain.Ports.Hub = 51821 + if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil || + !strings.Contains(err.Error(), "wg0") { + t.Fatalf("a tunnel not taken over stopped being refused for overlapping the range: %v", err) + } + + // Numbers that disagree with the tunnel are refused, naming the tunnel's. + for name, given := range map[string]Options{ + "--hub-port": {Adopted: true, Ports: FoundationPorts{Hub: 51821}, OverlayRange: DefaultOverlayRange}, + "--overlay-range": {Adopted: true, Ports: DefaultPorts(), OverlayRange: "10.77.0.0/16"}, + } { + if _, err := TakeTheTunnel(&given, m.run); err == nil || !strings.Contains(err.Error(), name) { + t.Errorf("a %s disagreeing with the tunnel was accepted: %v", name, err) + } + } + // And no tunnel up is an ordinary machine. + m.wg = "mesh0\n" + none := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange} + if found, err := TakeTheTunnel(&none, m.run); err != nil || found != nil || none.Ports.Hub != DefaultPorts().Hub { + t.Errorf("a machine with no tunnel was not left as it was: %+v %v", found, err) + } +} + +func aFoundKey() (string, error) { + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(k.Bytes()), nil +} diff --git a/internal/declaration/adoption_test.go b/internal/declaration/adoption_test.go index 1ca0d3b..e422516 100644 --- a/internal/declaration/adoption_test.go +++ b/internal/declaration/adoption_test.go @@ -105,3 +105,28 @@ func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) { t.Fatalf("a bundle claiming adoption was not refused: %v", err) } } + +// novox/hq ADR 0105: a service may take over a found tunnel, said whole and on an adopted node. +func TestTakingOverATunnelIsSaidWholeAndForARunningService(t *testing.T) { + adoptedWith := func(service string) error { + _, err := Parse([]byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[` + service + `]}`)) + return err + } + good := `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}` + if err := adoptedWith(good); err != nil { + t.Fatalf("a whole takeover on an adopted node was refused: %v", err) + } + for name, bad := range map[string]string{ + "its own unit": `{"id":"up","type":"service","unit":"wg-quick@wg0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + "no config": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0"}}`, + "a stopped service": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"stopped", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + } { + if err := adoptedWith(bad); err == nil { + t.Errorf("a takeover naming %s was accepted", name) + } + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index c0bc7af..237d6ae 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -617,6 +617,21 @@ type Service struct { // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). // A change that is also in RestartOn restarts it, which covers a reload. ReloadOn []string `json:"reload-on,omitempty"` + + // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit + // is started, the named unit is stopped and disabled — never flushed — and its configuration + // file is kept like any held file. Only on an adopted node, and only said by the controller, + // which knows the found tunnel's key is this node's own: without that, starting this unit on + // the found one's port would drop every peer's packets. + TakesOver *TakeOver `json:"takes-over,omitempty"` +} + +// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its +// configuration file. +type TakeOver struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` } func (s *Service) Identity() string { return s.ID } @@ -637,6 +652,19 @@ func (s *Service) validate(where string, _ bool) []string { "%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+ "leave the machine's own setting alone", where, s.Boot)) } + if t := s.TakesOver; t != nil { + switch { + case t.Unit == "" || t.Config == "" || t.Interface == "": + problems = append(problems, where+": takes-over names the found tunnel's interface, unit "+ + "and config, and this leaves one out") + case t.Unit == s.Unit: + problems = append(problems, fmt.Sprintf("%s: takes-over names %s, which is this service's own unit", + where, t.Unit)) + case s.State != "running": + problems = append(problems, where+": a service that takes over a tunnel is running — stopping "+ + "the found one for a service that will not run would leave the peers with nothing") + } + } return problems } @@ -1167,6 +1195,14 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { "resource %q: an opening is for an adopted node, and this declaration does not "+ "say the node is adopted", r.Identity())) } + if svc, ok := r.(*Service); ok && svc.TakesOver != nil { + // A tunnel is taken over on an adopted node, where what is found is kept: on a + // converged one there is nothing found to take over, and stopping a unit the + // mesh did not declare would be the host deciding (novox/hq ADR 0105). + problems = append(problems, fmt.Sprintf( + "resource %q: taking over a tunnel is for an adopted node, and this declaration "+ + "does not say the node is adopted", r.Identity())) + } } } diff --git a/internal/identity/overlay.go b/internal/identity/overlay.go index 4c3eb39..bc1342c 100644 --- a/internal/identity/overlay.go +++ b/internal/identity/overlay.go @@ -43,6 +43,27 @@ func GenerateOverlayKey() (OverlayKey, error) { }, nil } +// OverlayKeyFrom makes this node's overlay key from a private key it did not generate: the found +// tunnel's, on an adopted node whose private network takes that tunnel over (novox/hq ADR 0105). +// The one case where the mesh takes a credential it did not mint. From here on it is stored and +// sealed exactly as a generated one — in the identity file and the key file, readable by root +// alone — and the mesh receives only the public half, derived here from the private one so the +// two cannot disagree. +func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) { + raw, err := base64.StdEncoding.DecodeString(privateBase64) + if err != nil { + return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not base64: %w", err) + } + private, err := ecdh.X25519().NewPrivateKey(raw) + if err != nil { + return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not a Curve25519 key: %w", err) + } + return OverlayKey{ + Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(private.Bytes()), + }, nil +} + // OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface // configuration rather than embedded in it. // diff --git a/internal/identity/overlay_test.go b/internal/identity/overlay_test.go new file mode 100644 index 0000000..9ce8a76 --- /dev/null +++ b/internal/identity/overlay_test.go @@ -0,0 +1,28 @@ +package identity + +import ( + "strings" + "testing" +) + +// novox/hq ADR 0105: the found tunnel's private key becomes the node's overlay key, stored as a +// generated one is, and the public half the mesh records is derived from it — so the peers that +// know the tunnel by that key keep reaching it. +func TestAnOverlayKeyTakenFromAFoundTunnelIsTheSameKey(t *testing.T) { + generated, err := GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + taken, err := OverlayKeyFrom(generated.Private) + if err != nil { + t.Fatal(err) + } + if taken.Public != generated.Public || taken.Private != generated.Private { + t.Fatalf("a key taken from a private half is not that key: %+v vs %+v", taken, generated) + } + for _, bad := range []string{"", "not base64!", "c2hvcnQ="} { + if _, err := OverlayKeyFrom(bad); err == nil || !strings.Contains(err.Error(), "found tunnel") { + t.Errorf("%q was taken as a key: %v", bad, err) + } + } +} diff --git a/internal/link/enrol.go b/internal/link/enrol.go index 6cf839b..8f755c8 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -52,6 +52,30 @@ type EnrolRequest struct { // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish // on a token this key already spent (novox/hq issue 083). Proof []byte `json:"proof,omitempty"` + + // Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR + // 0105): everything about it but that key. Sent with the keys because it is one of them — + // OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the + // hub's address, the range and the carried peers from it before the first declaration. + Tunnel *Tunnel `json:"tunnel,omitempty"` +} + +// Tunnel is a found tunnel as it travels: no private key. +type Tunnel struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + Port int `json:"port"` + Address string `json:"address"` + Range string `json:"range"` + PublicKey string `json:"public_key"` + Peers []TunnelPeer `json:"peers,omitempty"` +} + +// TunnelPeer is one peer of a found tunnel: its key, and the address the tunnel routes to it. +type TunnelPeer struct { + PublicKey string `json:"public_key"` + Address string `json:"address"` } // EnrolReply is what the mesh says back. @@ -125,7 +149,7 @@ func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { // the broker who connected. func Enrol(ctx context.Context, address, pin, node, secret string, public []byte, overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, - timeout time.Duration) (EnrolReply, error) { + tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) { config, err := PinnedConfig(pin) if err != nil { @@ -172,7 +196,7 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, - Proof: proof} + Proof: proof, Tunnel: tunnel} body, err := json.Marshal(request) if err != nil { return EnrolReply{}, err diff --git a/internal/link/messages.go b/internal/link/messages.go index 953fe6c..fe27ccb 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -100,6 +100,21 @@ type Report struct { // published container port. Only an adopted node reports it; it is what converging the node // previews, so nothing closes without being named first. Reachable []Reach `json:"reachable,omitempty"` + + // Tunnel is what this adopted node says about the tunnel it found and carried (novox/hq ADR + // 0105): which interface, its port, range and peer count, whether the found interface is down + // and the mesh's up in its place, and where the found configuration's original was kept. + Tunnel *CarriedTunnel `json:"tunnel,omitempty"` +} + +// CarriedTunnel is this node's account of the tunnel it took over. +type CarriedTunnel struct { + Interface string `json:"interface"` + Port int `json:"port"` + Range string `json:"range"` + Peers int `json:"peers"` + Taken bool `json:"taken"` + Kept string `json:"kept,omitempty"` } // Held is one file or container found on an adopted node and kept as it was. diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go new file mode 100644 index 0000000..53c6bce --- /dev/null +++ b/internal/tunnel/tunnel.go @@ -0,0 +1,274 @@ +// Package tunnel reads the tunnel a predecessor left on a machine, so the mesh's private network +// can take it over in place (novox/hq ADR 0105). +// +// On an adopted node that is the hub, the mesh's interface is raised with the found interface's +// private key, on its port, with its address and range, and every peer it had. The found interface +// is stopped, never flushed; its configuration stays on disk. What this package does is the +// reading: which interface is there, what its file says, and what of that travels to the mesh — +// everything but the private key, which becomes the node's own overlay key and is stored the way +// that key is stored. +package tunnel + +import ( + "context" + "crypto/ecdh" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "sort" + "strconv" + "strings" +) + +// Runner executes a command. The same shape as everywhere else in this host. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// MeshInterface is the private network's own interface, which is never the found one. +const MeshInterface = "mesh0" + +// ConfigDir is where wg-quick keeps an interface's configuration. +const ConfigDir = "/etc/wireguard" + +// Found is a tunnel as found on the machine: everything the mesh is told about it, and the +// private key, which it is not. +type Found struct { + // Interface, Unit and Config are what the mesh's interface takes over. + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + // Port is the port the interface listens on; Address its own address with prefix length; + // Range the network that prefix names. + Port int `json:"port"` + Address string `json:"address"` + Range string `json:"range"` + // PublicKey is what every peer knows this tunnel by — derived here from the private key, so + // it is the key the file actually holds and not a comment beside it. + PublicKey string `json:"public_key"` + Peers []Peer `json:"peers,omitempty"` + + // privateKey never travels and never prints: not in JSON, not in %v. It is read once, to + // become the node's overlay key, and the file it came from is kept as found. + privateKey string +} + +// Peer is one peer of the found tunnel. +type Peer struct { + PublicKey string `json:"public_key"` + // Address is the one address the tunnel routes to the peer, as the file's AllowedIPs said it + // (with or without a /32). + Address string `json:"address"` + // Endpoint is where the found tunnel dialled the peer, if it did. Not carried to the mesh — + // a carried peer dials in, as it always did — but kept so a person reading the report sees + // what the file said. + Endpoint string `json:"endpoint,omitempty"` +} + +// PrivateKey is the found interface's private key, base64 as WireGuard writes it. The one +// accessor; a caller that has it is taking it as the node's key. +func (f Found) PrivateKey() string { return f.privateKey } + +// String is what a found tunnel prints as: never the key. +func (f Found) String() string { + return fmt.Sprintf("%s on port %d, %s in %s, %d peer(s)", f.Interface, f.Port, f.Address, + f.Range, len(f.Peers)) +} + +// MarshalJSON writes everything but the private key, whatever a caller passes to an encoder. +func (f Found) MarshalJSON() ([]byte, error) { + type wire Found + return json.Marshal(wire(f)) +} + +// ErrNone is a machine with no tunnel to take over. +var ErrNone = errors.New("no tunnel is up on this machine besides the mesh's own") + +// ErrSeveral is a machine with more than one, when nobody said which. +var ErrSeveral = errors.New("more than one tunnel is up on this machine") + +// ReadFile is how a configuration is read; a variable so a test can hand in a file. +var ReadFile = os.ReadFile + +// Find reads the tunnel to take over: the one named, or the one interface up besides the mesh's +// own. Nothing up is ErrNone — an ordinary answer, the machine has no tunnel to adopt — and two +// or more with none named is ErrSeveral, naming them, because choosing would be deciding. +// +// Read from the interface's configuration file rather than from the running interface: the file +// is what wg-quick raised and what carries the address, which the kernel does not report per +// interface the way the key and peers are. The running interface is consulted only to know the +// tunnel is up — a file for an interface nothing runs is not a tunnel the peers are reaching. +func Find(ctx context.Context, run Runner, named string) (Found, error) { + out, err := run(ctx, "wg", "show", "interfaces") + if err != nil { + return Found{}, fmt.Errorf("cannot ask which tunnels are up on this machine: %w", err) + } + var up []string + for _, iface := range strings.Fields(out) { + if iface != MeshInterface { + up = append(up, iface) + } + } + sort.Strings(up) + iface := named + switch { + case named != "": + found := false + for _, u := range up { + if u == named { + found = true + } + } + if !found { + return Found{}, fmt.Errorf("%s was named as the tunnel to take over and is not up; up: %s", + named, orNone(up)) + } + case len(up) == 0: + return Found{}, ErrNone + case len(up) > 1: + return Found{}, fmt.Errorf("%w: %s. Name the one the predecessor's machines reach with --tunnel", + ErrSeveral, strings.Join(up, ", ")) + default: + iface = up[0] + } + + path := ConfigDir + "/" + iface + ".conf" + raw, err := ReadFile(path) + if err != nil { + return Found{}, fmt.Errorf("%s is up and its configuration cannot be read: %w", iface, err) + } + found, err := Parse(raw) + if err != nil { + return Found{}, fmt.Errorf("%s: %w", path, err) + } + found.Interface, found.Unit, found.Config = iface, "wg-quick@"+iface, path + return found, nil +} + +func orNone(names []string) string { + if len(names) == 0 { + return "none" + } + return strings.Join(names, ", ") +} + +// Parse reads a wg-quick configuration: the interface's key, port and address, and each peer's +// key and allowed address. Refused when it lacks what the mesh needs — a key, an address with a +// prefix — because a tunnel taken over without them is one the peers cannot reach. +func Parse(raw []byte) (Found, error) { + var f Found + section := "" + var peer *Peer + closePeer := func() error { + if peer == nil { + return nil + } + if peer.PublicKey == "" { + return errors.New("a [Peer] section has no PublicKey") + } + if peer.Address == "" { + return fmt.Errorf("the peer %s has no AllowedIPs, so the tunnel routes nothing to it", + short(peer.PublicKey)) + } + f.Peers = append(f.Peers, *peer) + peer = nil + return nil + } + for n, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if i := strings.IndexAny(line, "#;"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + if strings.HasPrefix(line, "[") { + if err := closePeer(); err != nil { + return Found{}, err + } + section = strings.ToLower(strings.Trim(line, "[]")) + if section == "peer" { + peer = &Peer{} + } + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + return Found{}, fmt.Errorf("line %d is not `key = value`", n+1) + } + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + switch section { + case "interface": + switch key { + case "privatekey": + f.privateKey = value + case "listenport": + port, err := strconv.Atoi(value) + if err != nil || port < 1 || port > 65535 { + return Found{}, fmt.Errorf("ListenPort %q is not a port", value) + } + f.Port = port + case "address": + // The first address is the interface's; a second family would be a second + // tunnel's worth of addressing, which this does not carry. + first := strings.TrimSpace(strings.Split(value, ",")[0]) + ip, network, err := net.ParseCIDR(first) + if err != nil { + return Found{}, fmt.Errorf("Address %q is not an address with a prefix length, "+ + "and the range the mesh takes over is read from the prefix", first) + } + f.Address = first + f.Range = network.String() + _ = ip + } + case "peer": + switch key { + case "publickey": + peer.PublicKey = value + case "allowedips": + peer.Address = strings.TrimSpace(strings.Split(value, ",")[0]) + case "endpoint": + peer.Endpoint = value + } + } + } + if err := closePeer(); err != nil { + return Found{}, err + } + if f.privateKey == "" { + return Found{}, errors.New("no PrivateKey in [Interface]; the mesh takes a tunnel over with its key or not at all") + } + if f.Address == "" { + return Found{}, errors.New("no Address in [Interface], so neither the hub's address nor the range can be read") + } + if f.Port == 0 { + return Found{}, errors.New("no ListenPort in [Interface]: a tunnel with no port is one nothing dials, so there is nothing to take over") + } + public, err := PublicKeyOf(f.privateKey) + if err != nil { + return Found{}, err + } + f.PublicKey = public + return f, nil +} + +// PublicKeyOf derives the public half of a WireGuard private key, both base64. +func PublicKeyOf(privateBase64 string) (string, error) { + raw, err := base64.StdEncoding.DecodeString(privateBase64) + if err != nil { + return "", fmt.Errorf("the private key is not base64: %w", err) + } + private, err := ecdh.X25519().NewPrivateKey(raw) + if err != nil { + return "", fmt.Errorf("the private key is not a Curve25519 key: %w", err) + } + return base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), nil +} + +func short(key string) string { + if len(key) > 8 { + return key[:8] + "…" + } + return key +} diff --git a/internal/tunnel/tunnel_test.go b/internal/tunnel/tunnel_test.go new file mode 100644 index 0000000..7272a6b --- /dev/null +++ b/internal/tunnel/tunnel_test.go @@ -0,0 +1,172 @@ +package tunnel + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" +) + +// novox/hq ADR 0105: the host reads the predecessor's tunnel — key, port, address and range, every +// peer — and the private key becomes the node's, never printed and never sent. + +// aKey is a real WireGuard keypair, made here so a key that stopped being a key is caught. +func aKey(t *testing.T) (private, public string) { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(k.Bytes()), + base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()) +} + +func aConfig(private string, peers ...string) string { + var b strings.Builder + fmt.Fprintf(&b, "# the predecessor's hub\n[Interface]\nPrivateKey = %s\nListenPort = 51900\n"+ + "Address = 192.0.2.1/24\n", private) + for i, key := range peers { + fmt.Fprintf(&b, "\n[Peer]\nPublicKey = %s\nAllowedIPs = 192.0.2.%d/32\n", key, i+2) + } + return b.String() +} + +func TestTheConfigurationIsReadWhole(t *testing.T) { + private, public := aKey(t) + _, peerA := aKey(t) + _, peerB := aKey(t) + found, err := Parse([]byte(aConfig(private, peerA, peerB) + "PersistentKeepalive = 25 ; a comment\n")) + if err != nil { + t.Fatal(err) + } + if found.Port != 51900 || found.Address != "192.0.2.1/24" || found.Range != "192.0.2.0/24" { + t.Errorf("port, address or range misread: %+v", found) + } + if found.PublicKey != public { + t.Errorf("the public key is not the one derived from the file's private key") + } + if found.PrivateKey() != private { + t.Error("the private key was not read") + } + if len(found.Peers) != 2 || found.Peers[0].PublicKey != peerA || found.Peers[0].Address != "192.0.2.2/32" || + found.Peers[1].PublicKey != peerB || found.Peers[1].Address != "192.0.2.3/32" { + t.Errorf("the peers were misread: %+v", found.Peers) + } +} + +func TestThePrivateKeyNeverPrintsAndNeverTravels(t *testing.T) { + private, _ := aKey(t) + found, err := Parse([]byte(aConfig(private))) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(found) + if err != nil { + t.Fatal(err) + } + for what, said := range map[string]string{ + "JSON": string(raw), + "String": found.String(), + "%v": fmt.Sprintf("%v", found), + "%+v": fmt.Sprintf("%+v", found), + "%#v via %v": fmt.Sprintf("%v", []Found{found}), + } { + if strings.Contains(said, private) { + t.Errorf("the private key appears in %s: %s", what, said) + } + } + if !strings.Contains(string(raw), found.PublicKey) { + t.Error("the public key does not travel, so the mesh could not know the tunnel's key") + } +} + +func TestATunnelWithoutWhatTheMeshNeedsIsRefused(t *testing.T) { + private, _ := aKey(t) + _, peer := aKey(t) + for name, conf := range map[string]string{ + "no key": "[Interface]\nListenPort = 51900\nAddress = 192.0.2.1/24\n", + "no address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\n", private), + "bare address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\nAddress = 192.0.2.1\n", private), + "no port": fmt.Sprintf("[Interface]\nPrivateKey = %s\nAddress = 192.0.2.1/24\n", private), + "peer no route": aConfig(private) + "\n[Peer]\nPublicKey = " + peer + "\n", + "peer no key": aConfig(private) + "\n[Peer]\nAllowedIPs = 192.0.2.9/32\n", + "not a key": "[Interface]\nPrivateKey = not-base64!\nListenPort = 1\nAddress = 192.0.2.1/24\n", + } { + if _, err := Parse([]byte(conf)); err == nil { + t.Errorf("%s was accepted", name) + } + } +} + +// aMachine answers `wg show interfaces` and reads configurations from a map. +type aMachine struct { + up string + files map[string]string + asked []string +} + +func (m *aMachine) run(_ context.Context, name string, args ...string) (string, error) { + m.asked = append(m.asked, name+" "+strings.Join(args, " ")) + if name == "wg" && len(args) == 2 && args[0] == "show" && args[1] == "interfaces" { + return m.up, nil + } + return "", errors.New("unexpected: " + name) +} + +func (m *aMachine) read(path string) ([]byte, error) { + if raw, ok := m.files[path]; ok { + return []byte(raw), nil + } + return nil, errors.New("no such file: " + path) +} + +func TestTheOneTunnelUpBesidesTheMeshsIsFound(t *testing.T) { + private, public := aKey(t) + m := &aMachine{up: "mesh0 wg0\n", files: map[string]string{ConfigDir + "/wg0.conf": aConfig(private)}} + ReadFile = m.read + t.Cleanup(func() { ReadFile = os.ReadFile }) + + found, err := Find(context.Background(), m.run, "") + if err != nil { + t.Fatal(err) + } + if found.Interface != "wg0" || found.Unit != "wg-quick@wg0" || found.Config != ConfigDir+"/wg0.conf" || + found.PublicKey != public { + t.Errorf("the wrong tunnel, or misnamed: %+v", found) + } + for _, asked := range m.asked { + if strings.HasPrefix(asked, "wg set") || strings.Contains(asked, "private-key") { + t.Errorf("finding a tunnel ran %q; reading is reading", asked) + } + } +} + +func TestNoneUpIsAnOrdinaryAnswerAndSeveralIsAQuestion(t *testing.T) { + private, _ := aKey(t) + m := &aMachine{up: "mesh0\n", files: map[string]string{ + ConfigDir + "/wg0.conf": aConfig(private), ConfigDir + "/wg1.conf": aConfig(private)}} + ReadFile = m.read + t.Cleanup(func() { ReadFile = os.ReadFile }) + + if _, err := Find(context.Background(), m.run, ""); !errors.Is(err, ErrNone) { + t.Errorf("a machine with only the mesh's interface up was not an ordinary none: %v", err) + } + m.up = "wg1 mesh0 wg0\n" + _, err := Find(context.Background(), m.run, "") + if !errors.Is(err, ErrSeveral) || !strings.Contains(err.Error(), "wg0, wg1") || strings.Contains(err.Error(), "mesh0") { + t.Errorf("two tunnels up were not refused naming both and only them: %v", err) + } + found, err := Find(context.Background(), m.run, "wg1") + if err != nil || found.Interface != "wg1" { + t.Errorf("naming one of two did not find it: %+v %v", found, err) + } + if _, err := Find(context.Background(), m.run, "wg9"); err == nil || !strings.Contains(err.Error(), "wg9") { + t.Errorf("naming a tunnel that is not up was not refused: %v", err) + } +} -- 2.54.0 From fc593b9dfd88f39ab7660f5d850b611768d09f37 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 24 Sep 2026 00:02:08 +0200 Subject: [PATCH 2/2] Stop nothing the mesh cannot replace, give the tunnel back on failure, and take it over after enrolment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the ADR 0105 build (hq ADR 0105). The takeover stopped the found unit and then found out whether the mesh's interface would do; a start that failed left the machine with no tunnel at all. Now nothing is stopped until the declared interface listens on the found port at the found address and the key file it names holds the found key — the refusal names the remedy — and a mesh interface that fails to start after the takeover has the found unit started again, with the account saying so. The account has three states (not taken, taken, down) and is given on every takeover, failure included. An interface raised by hand is looked at again for a moment and then refused naming `wg-quick down`. A found unit started again by hand beside the mesh's is said, not stopped: on the hub it cannot hold the port, and on a spoke two interfaces with one key would fight. `mesh-host overlay take --tunnel ` is the path for a node that enrolled before the mesh knew to take a tunnel over: the found key becomes its overlay key — identity, sealing and serving keys untouched, so nothing sealed to the node is remade — and the mesh is told with a rekey signed by the identity key, over the key left, the key taken and the tunnel. Told first, written second, so a run again puts right whichever half did not happen. --- cmd/mesh-host/main.go | 87 ++++++++++- cmd/mesh-host/rekey_test.go | 94 +++++++++++ internal/apply/apply.go | 29 +++- internal/apply/hold_test.go | 6 +- internal/apply/takeover.go | 266 +++++++++++++++++++++++++++----- internal/apply/takeover_test.go | 157 +++++++++++++++---- internal/identity/identity.go | 7 +- internal/link/messages.go | 56 ++++++- internal/link/run.go | 33 ++++ 9 files changed, 655 insertions(+), 80 deletions(-) create mode 100644 cmd/mesh-host/rekey_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index a8f420a..b38c6a1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -56,6 +56,8 @@ const usage = `mesh-host — the node host apply FILE make this machine match a declaration from a file reconcile make this machine match the declaration this host carries bundle show what this host carries + overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this + node's overlay key and the mesh is told, signed; --tunnel when several are up owned what this host has applied and still owns version @@ -147,6 +149,13 @@ func parseArgs(args []string) (string, options, error) { opts.file = positionals[0] return command, opts, nil } + if command == "overlay" { + if len(positionals) != 1 { + return "", opts, errors.New("overlay take [--tunnel ]") + } + opts.file = positionals[0] + return command, opts, nil + } // Anything left over was neither the command nor a flag. Refused rather than ignored: a // mistyped argument that changes nothing and reports success is worse than an error. if len(positionals) > 0 { @@ -234,6 +243,8 @@ func run(ctx context.Context, command string, opts options) error { case "enrol", "enroll": return enrol(ctx, opts) + case "overlay": + return overlayCommand(ctx, opts) case "run": return runLink(ctx, opts) @@ -608,6 +619,80 @@ func enrol(ctx context.Context, opts options) error { return nil } +// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over +// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a +// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them +// every credential the mesh sealed to it. +func overlayCommand(ctx context.Context, opts options) error { + if opts.file != "take" { + return errors.New("overlay take [--tunnel ] — the one thing `overlay` does here") + } + identityPath := identity.Path(opts.state) + mine, err := identity.Load(identityPath) + if err != nil { + return err + } + found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) + if err != nil { + return fmt.Errorf("%w. Nothing was changed", err) + } + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + return err + } + fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public) + fmt.Printf(" identity, sealing and serving keys are untouched\n") + + // Told first, then written: a mesh told and a machine not yet written is put right by running + // this again (the mesh refuses the stale second rekey and changes nothing; the files are + // rewritten the same). A machine written and a mesh not told would raise the mesh's interface + // on a key the mesh does not know at the next restart. + if err := link.Publish(ctx, link.Membership{ + Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, + Password: mine.Membership.Password, Signer: mine.Membership.Signer, + }, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil { + return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err) + } + fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node) + + if err := os.WriteFile(identity.OverlayKeyPath(opts.state), + []byte(taken.Overlay.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err) + } + if err := identity.Save(identityPath, taken); err != nil { + return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err) + } + fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n", + identity.OverlayKeyPath(opts.state)) + fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint :%d …`, `plan %s --json`, then push\n", + mine.Node, found.Port, mine.Node) + return nil +} + +// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey +// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same +// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names +// is the one before the take, so the mesh can tell a repeat from a replay. +func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) { + overlay, err := identity.OverlayKeyFrom(found.PrivateKey()) + if err != nil { + return identity.Identity{}, link.Rekey{}, err + } + previous := mine.Overlay.Public + if previous == overlay.Public && mine.OverlayBefore != "" { + previous = mine.OverlayBefore + } + taken := mine + taken.Overlay = overlay + if previous != overlay.Public { + taken.OverlayBefore = previous + } + presented := carried(found) + rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented} + rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented)) + return taken, rekey, nil +} + // carried is a found tunnel as it is presented to the mesh: everything but its private key. func carried(t tunnel.Found) *link.Tunnel { out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, @@ -934,7 +1019,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105). if t := outcome.Tunnel; t != nil { report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range, - Peers: t.Peers, Taken: t.Taken, Kept: t.Kept} + Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept} } reached, err := reachable.Collect(ctx, apply.ExecRunner) if err != nil { diff --git a/cmd/mesh-host/rekey_test.go b/cmd/mesh-host/rekey_test.go new file mode 100644 index 0000000..7b19edd --- /dev/null +++ b/cmd/mesh-host/rekey_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/ed25519" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/tunnel" +) + +// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and +// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a +// proof signed by the identity key, over the previous key, the new one and the tunnel. + +func anEnrolledNode(t *testing.T) identity.Identity { + t.Helper() + mine, err := identity.Generate("anchor") + if err != nil { + t.Fatal(err) + } + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa", + Signer: make([]byte, ed25519.PublicKeySize), Password: "p"} + return mine +} + +func aFoundTunnel(t *testing.T) tunnel.Found { + t.Helper() + private, err := identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" + + "Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n")) + if err != nil { + t.Fatal(err) + } + found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf" + return found +} + +func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) { + mine := anEnrolledNode(t) + found := aFoundTunnel(t) + before := mine.Overlay.Public + + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + t.Fatal(err) + } + if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() { + t.Fatal("the overlay key is not the tunnel's") + } + if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) || + taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password || + taken.Membership.Broker != mine.Membership.Broker { + t.Fatal("something other than the overlay key moved") + } + if taken.OverlayBefore != before { + t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore) + } + if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil || + rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 { + t.Fatalf("the rekey does not say what moved: %+v", rekey) + } + if !ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the rekey is not signed by this node's identity key over what it says") + } + if ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the proof is not bound to the node") + } + for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} { + if strings.Contains(said, found.PrivateKey()) { + t.Fatal("the private key travels") + } + } + + // Run again after the take — the mesh not yet told, or told and refused — the previous key it + // names is still the one before the take, so the mesh can tell a repeat from a replay. + again, second, err := rekeyOnto(taken, found) + if err != nil { + t.Fatal(err) + } + if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey { + t.Fatalf("a take run again does not name the key before the first: %+v", second) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index c2fd918..6c4309b 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -339,22 +339,32 @@ func ApplyKeeping( // it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never // flushed. A failure here fails the service too — the mesh's interface is not started on a // port the found one still holds. + stoppedFound := false if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { var outcome Outcome var facts TakenTunnel var err error if d.Adoption == nil { err = errNotAdopted + facts = TakenTunnel{Interface: svc.TakesOver.Interface, State: NotTaken} } else { - outcome, facts, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) + outcome, facts, stoppedFound, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) } + // Always an account, failure included: the last account standing must never be an + // older "taken" over a machine whose takeover has since gone wrong. + report.Tunnel = &facts if err != nil { + report.Tunnel.Note = err.Error() + if stoppedFound { + // The found unit is down and the mesh's not up: the one state where the + // peers reach nothing. Started again, and said. + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report}) log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) continue } report.Outcomes = append(report.Outcomes, outcome) - report.Tunnel = &facts log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) } @@ -377,6 +387,17 @@ func ApplyKeeping( failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { + // The mesh's interface did not come up after the found one was stopped: no + // tunnel at all. The found unit is started again — the machine goes back to + // what it had — and the account says so (novox/hq ADR 0105). + report.Tunnel.Note = "the mesh's interface did not come up: " + err.Error() + if stoppedFound { + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } else { + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + } + } // **A failed action stops what follows. Nothing else does.** // @@ -426,8 +447,8 @@ func ApplyKeeping( } if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { // The found interface is down and the mesh's is up in its place: the tunnel changed - // hands (novox/hq ADR 0105). - report.Tunnel.Taken = true + // hands (novox/hq ADR 0105). Read from the machine, not assumed. + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 3e95e36..be2277c 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -31,6 +31,8 @@ type machine struct { type fakeUnit struct { active, enabled string + // wontStart is a unit that accepts `start` and stays inactive — one that starts and dies. + wontStart bool // fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an // administrator installs one. fragment string @@ -65,7 +67,9 @@ func (m *machine) systemctl(args []string) (string, error) { } return u.enabled + "\n", nil case "start": - u.active = "active" + if !u.wontStart { + u.active = "active" + } case "stop": u.active = "inactive" case "enable": diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index bff4935..359412c 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -34,26 +34,53 @@ type TakenTunnel struct { Port int Range string Peers int - // Taken is whether the found interface is down and disabled and the mesh's up in its place. - Taken bool + // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one + // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's + // not up: the peers reach nothing). Note is what this apply did about it. + State string + Note string Kept string } +// The states, as the link says them. +const ( + NotTaken = "not-taken" + Taken = "taken" + TunnelDown = "down" +) + +// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up +// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a +// person takes a moment. Variables so a test need not wait. +var ( + takeoverRecheck = 2 * time.Second + takeoverRechecks = 3 +) + // takeOverID is the held record's id for the found configuration: the service's own with a suffix, // so it is declared for as long as the service is and never mistaken for the service itself. func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" } // takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that // replaces it. Returned is the hold's outcome, and what was found for the report. +// +// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's +// option 2 is exactly this going wrong): the declared configuration must listen on the found port +// at the found address, and the key file it points at must hold the found key. Only then is the +// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then +// fails to start can have the found unit started again. func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration, - known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) { + known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) { t := svc.TakesOver id := takeOverID(svc) module, _ := d.Adoption.UntakenModuleOf(svc.ID) if module == "" { module = "the private network" } - facts := TakenTunnel{Interface: t.Interface} + facts = TakenTunnel{Interface: t.Interface, State: NotTaken} + + // 0. What the found configuration says, before anything: the checks below are against it. + found, ferr := readFoundTunnel(t.Config) // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. @@ -62,65 +89,97 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, out, held, err := hold(ctx, sys, file, module, was, already, "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) if err != nil { - return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) } known.RecordHeld(held) facts.Kept = held.Kept - // What the file says, for the report: read from the machine, or from the kept original when - // the machine's copy is gone. The private key stays in the file; nothing here keeps it. + // What the file says, for the report: from the machine, or from the kept original when the + // machine's copy is gone. The private key stays in the file; nothing here keeps it. unread := "" - raw, err := os.ReadFile(t.Config) - if err != nil && held.Kept != "" { - raw, err = os.ReadFile(held.Kept) + if ferr != nil && held.Kept != "" { + found, ferr = readFoundTunnel(held.Kept) } - if err == nil { - if found, perr := tunnel.Parse(raw); perr == nil { - facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) - } else { - unread = perr.Error() - } + if ferr == nil { + facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) } else { - unread = err.Error() + unread = ferr.Error() } - // 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not - // there is not an error — the interface may have been raised another way, which the check - // below catches — and neither is one already down. + // 2. Where things stand: the found unit, and the mesh's. + foundState, unitErr := sys.ServiceState(ctx, run, t.Unit) + meshState, _ := sys.ServiceState(ctx, run, svc.Unit) + if foundState == "running" && meshState == "running" { + // Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's + // holds — and on a spoke two interfaces with one key flap between them. Not stopped again + // by the mesh: what is found on an adopted node is reported, and the first takeover was + // the one act (the PR note says why). Said, so a person sees it. + facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " + + "`systemctl stop " + t.Unit + "` on the machine" + } + + // 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared + // configuration must listen on the found port at the found address, and the key file it + // points at must hold the found key, or the peers would be dropped the moment it came up. + if foundState == "running" && meshState != "running" { + if ferr != nil { + return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+ + "tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running", + t.Config, ferr, t.Unit) + } + if err := replaces(d, svc, found); err != nil { + return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit) + } + } + + // 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at + // boot. A unit that is not there is not an error — the interface may have been raised + // another way, which the check below catches — and neither is one already down. var did []string - state, err := sys.ServiceState(ctx, run, t.Unit) switch { - case err != nil: + case unitErr != nil: did = append(did, t.Unit+" is not a unit here") - case state == "running": + case foundState == "running" && meshState != "running": if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil { - return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err) + return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err) } after, err := sys.ServiceState(ctx, run, t.Unit) if err != nil { - return out, facts, err + return out, facts, true, err } if after != "stopped" { - return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) + return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) } + stopped = true did = append(did, "stopped "+t.Unit) } - if err == nil { + if unitErr == nil { if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" { if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil { - return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) + return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) } did = append(did, "disabled it at boot") } } - // 3. The interface is gone. If it is still up, something other than its unit raised it, and - // starting the mesh's on the same port and address would fail or, worse, half work. - if up, err := run(ctx, "wg", "show", "interfaces"); err == nil { - for _, iface := range strings.Fields(up) { - if iface == t.Interface { - return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+ - "than that unit raises it, and the mesh's interface cannot take its port and address "+ - "while it does. Nothing was flushed", t.Interface, t.Unit) + // 5. The interface is gone. If it is still up, something other than its unit raised it — + // the predecessor brings its up by hand — and the mesh's interface cannot take its port + // and address while it is. Looked at again for a moment, since a person taking it down + // takes a moment; then refused, naming what to do. + if meshState != "running" { + for try := 0; ; try++ { + if !interfaceUp(ctx, run, t.Interface) { + break + } + if try >= takeoverRechecks { + return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+ + "raised by hand, not by its unit, and the mesh's interface cannot take its port and "+ + "address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+ + "over. Nothing was flushed", t.Interface, t.Unit, t.Interface) + } + select { + case <-ctx.Done(): + return out, facts, stopped, ctx.Err() + case <-time.After(takeoverRecheck): } } } @@ -140,7 +199,140 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // peers was carried, which reads as a tunnel that was not one. out.Detail += "; what it says could not be read as a tunnel's: " + unread } - return out, facts, nil + return out, facts, stopped, nil +} + +// readFoundTunnel is the found configuration as a tunnel. +func readFoundTunnel(path string) (tunnel.Found, error) { + raw, err := os.ReadFile(path) + if err != nil { + return tunnel.Found{}, err + } + return tunnel.Parse(raw) +} + +// interfaceUp is whether a WireGuard interface is up on the machine. +func interfaceUp(ctx context.Context, run Runner, iface string) bool { + up, err := run(ctx, "wg", "show", "interfaces") + if err != nil { + return false + } + for _, name := range strings.Fields(up) { + if name == iface { + return true + } + } + return false +} + +// replaces holds the mesh's declared interface configuration against the found tunnel it is to +// replace: same port, same address, and a key file holding the found key. The configuration is +// the file the service restarts on; its `PostUp = wg set %i private-key ` names the key. +func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error { + var conf *declaration.File + for _, r := range d.Resources { + f, ok := r.(*declaration.File) + if !ok { + continue + } + for _, id := range svc.RestartOn { + if f.ID == id { + conf = f + } + } + } + if conf == nil { + return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+ + "raise cannot be checked against the found one", svc.Unit, found.Interface) + } + port, address, keyPath := "", "", "" + for _, line := range strings.Split(conf.Content, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + switch key { + case "listenport": + port = value + case "address": + address = strings.TrimSpace(strings.Split(value, ",")[0]) + case "postup": + if _, after, ok := strings.Cut(value, "private-key "); ok { + keyPath = strings.Fields(after)[0] + } + } + } + var wrong []string + if port != fmt.Sprint(found.Port) { + wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port)) + } + if host(address) != host(found.Address) { + wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address)) + } + switch raw, err := os.ReadFile(keyPath); { + case keyPath == "": + wrong = append(wrong, "it names no key file") + case err != nil: + wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err)) + default: + public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw))) + if perr != nil || public != found.PublicKey { + wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+ + "--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface)) + } + } + if len(wrong) > 0 { + return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+ + "dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again", + found.Interface, strings.Join(wrong, "; ")) + } + return nil +} + +// host is an address without its prefix length. +func host(address string) string { + if i := strings.Index(address, "/"); i >= 0 { + return address[:i] + } + return address +} + +// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up +// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is +// down — the peers reach nothing. +func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string { + foundState, _ := sys.ServiceState(ctx, run, foundUnit) + meshState, _ := sys.ServiceState(ctx, run, meshUnit) + foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@")) + switch { + case meshState == "running" && !foundUp: + return Taken + case meshState == "running": + // Both up: not a takeover that holds, and said as not taken so nobody reads it as one. + return NotTaken + case foundUp: + return NotTaken + default: + return TunnelDown + } +} + +// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the +// machine has the tunnel it had rather than none, and says so in the account. +func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) { + if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil { + facts.State = TunnelDown + facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit + return + } + if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" { + facts.State = TunnelDown + facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit + return + } + facts.State = NotTaken + facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 2ca9abf..18fde16 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -14,53 +14,63 @@ import ( ) // novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the -// found interface without flushing it, and keeps its configuration. +// found interface without flushing it, and keeps its configuration — and stops nothing until the +// mesh's interface is known to be able to replace it. -// foundConf is the predecessor's configuration, with a real key made once per run: the key is -// what the takeover must never print or copy, so it had better be one. -var foundConf = func() string { +// foundKey is the predecessor's private key, a real one made once per run: the key is what the +// takeover must never print or copy, so it had better be one. +var foundKey = func() string { k, err := ecdh.X25519().GenerateKey(rand.Reader) if err != nil { panic(err) } - return "[Interface]\nPrivateKey = " + base64.StdEncoding.EncodeToString(k.Bytes()) + "\n" + - "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + - "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" + return base64.StdEncoding.EncodeToString(k.Bytes()) }() +var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" + + "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + + "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" + // aTakeover is the private network's declaration for an adopted hub whose interface takes over -// the found tunnel: the mesh's configuration — with the found key set from the node's own key file -// and the found peers in its list — and the interface's service naming what it replaces. -func aTakeover(t *testing.T, config, mesh string) *declaration.Declaration { +// the found tunnel: the mesh's configuration — on the found port and address, its key set from the +// node's own key file, the found peers in its list — and the interface's service naming what it +// replaces. Port and address are parameters so a test can declare a wrong one. +func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration { t.Helper() return adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`, `{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600", - "content":"[Interface]\nAddress = 192.0.2.1/32\nListenPort = 51900\nPostUp = wg set %i private-key /var/lib/mesh-host/overlay.key\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, + "content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled", "restart-on":["mesh-wireguard.overlay-config"], "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`) } -// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet. -func aHubInUse(t *testing.T) (dir, config, mesh string, m *machine) { +// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found +// configuration on disk, and the node's key file holding the found key, as enrolment left it. +func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) { t.Helper() dir = t.TempDir() config = filepath.Join(dir, "wg0.conf") mesh = filepath.Join(dir, "mesh0.conf") + keyFile = filepath.Join(dir, "overlay.key") if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil { t.Fatal(err) } + if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil { + t.Fatal(err) + } m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{ "wg-quick@wg0": {active: "active", enabled: "enabled"}, "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, }} - return dir, config, mesh, m + takeoverRecheck = 0 + return dir, config, mesh, keyFile, m } func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) - report, state := applyAdopted(t, aTakeover(t, config, mesh), store.State{}, m, dir) + dir, config, mesh, keyFile, m := aHubInUse(t) + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) // The found interface: its unit stopped and disabled, and nothing else done to it. if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { @@ -93,16 +103,18 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got) } // And the report says so, with what was found — port, range, peers — and never the key. - if report.Tunnel == nil || !report.Tunnel.Taken || report.Tunnel.Port != 51900 || + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 || report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept { t.Fatalf("the report does not say what was carried: %+v", report.Tunnel) } - private := strings.TrimSpace(strings.SplitN(strings.SplitN(foundConf, "PrivateKey = ", 2)[1], "\n", 2)[0]) for _, o := range report.Outcomes { - if strings.Contains(o.Detail, private) { + if strings.Contains(o.Detail, foundKey) { t.Errorf("the found key was printed in an outcome: %+v", o) } } + if strings.Contains(report.Tunnel.Note, foundKey) { + t.Error("the found key was printed in the account") + } if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" || !strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") { t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o) @@ -112,9 +124,67 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T } } -func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) - d := aTakeover(t, config, mesh) +func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + otherKey := filepath.Join(dir, "other.key") + k, _ := ecdh.X25519().GenerateKey(rand.Reader) + if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cases := map[string]*declaration.Declaration{ + "another port": aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"), + "another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"), + "another key": aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"), + "no key file": aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"), + } + for name, d := range cases { + m.asked = nil + report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{}, + store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") { + t.Fatalf("%s: the takeover was not refused: %v", name, err) + } + if name == "another key" && !strings.Contains(err.Error(), "overlay take") { + t.Errorf("%s: the refusal does not name the remedy: %v", name, err) + } + if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") { + t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name) + } + if m.units["wg-quick@mesh0"].active == "active" { + t.Fatalf("%s: the mesh's interface was started on top of the found one", name) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") { + t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel) + } + if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { + t.Errorf("%s: the found configuration was not kept before the refusal", name) + } + } +} + +func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.units["wg-quick@mesh0"].wontStart = true + report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil { + t.Fatal("a mesh interface that did not come up was reported as applied") + } + if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") { + t.Fatalf("the found unit was not stopped and then started again: %v", m.asked) + } + if m.units["wg-quick@wg0"].active != "active" { + t.Fatal("the machine was left with no tunnel at all") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || + !strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") { + t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel) + } +} + +func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") _, state := applyAdopted(t, d, store.State{}, m, dir) m.asked = nil @@ -128,28 +198,49 @@ func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) { if m.did("systemctl stop wg-quick@wg0") { t.Error("a found unit already down was stopped again") } + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel) + } - // Somebody starts the found unit again: it would take the port back, so it is stopped again - // — the one thing on an adopted node the mesh undoes, because the tunnel is the mesh's now. + // Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh — + // on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said. m.units["wg-quick@wg0"].active = "active" m.asked = nil - _, _ = applyAdopted(t, d, again, m, dir) - if m.units["wg-quick@wg0"].active != "inactive" || !m.did("systemctl stop wg-quick@wg0") { - t.Error("a found unit started again was left holding the mesh's port") + report, _ = applyAdopted(t, d, again, m, dir) + if m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit started again by hand was stopped by the mesh") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") { + t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel) } } -func TestAFoundInterfaceStillUpAfterItsUnitStoppedRefusesTheTakeover(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) +func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + // The unit is not running, yet the interface is up: the predecessor raised it by hand. + m.units["wg-quick@wg0"].active = "inactive" m.wgUp = "wg0 mesh0\n" - _, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh), store.State{}, - store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) - if err == nil || !strings.Contains(err.Error(), "still up") || !strings.Contains(err.Error(), "Nothing was flushed") { - t.Fatalf("an interface something else raises was taken over anyway: %v", err) + report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") { + t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err) } if m.units["wg-quick@mesh0"].active == "active" { t.Error("the mesh's interface was started on a port the found one still holds") } + // Looked at more than once before giving up: a person taking it down takes a moment. + shows := 0 + for _, a := range m.asked { + if a == "wg show interfaces" { + shows++ + } + } + if shows < takeoverRechecks+1 { + t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken { + t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel) + } if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { t.Error("the found configuration was not kept before the refusal") } diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 9ff2ee8..c8d7430 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -49,8 +49,13 @@ type Identity struct { Membership Membership `json:"membership"` // Overlay is this node's key on the private network. Generated here, like the identity above, - // and for the same reason: the mesh computes a graph it cannot impersonate. + // and for the same reason: the mesh computes a graph it cannot impersonate — or, on an adopted + // node that took a found tunnel over, that tunnel's key (novox/hq ADR 0105). Overlay OverlayKey `json:"overlay"` + // OverlayBefore is the public half of the overlay key this node held before it took a found + // tunnel's, so a take run again names the key the mesh still records. Empty on a node that + // never took one. + OverlayBefore string `json:"overlay_before,omitempty"` } // Membership is how this node reaches the mesh it belongs to, and who it believes. diff --git a/internal/link/messages.go b/internal/link/messages.go index fe27ccb..76765a0 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,6 +1,10 @@ package link -import "time" +import ( + "strconv" + "strings" + "time" +) // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field @@ -105,18 +109,64 @@ type Report struct { // 0105): which interface, its port, range and peer count, whether the found interface is down // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + + // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq + // ADR 0105). Not an account of the machine: a report carrying one says nothing else. + Rekey *Rekey `json:"rekey,omitempty"` } -// CarriedTunnel is this node's account of the tunnel it took over. +// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried +// states below; Note is what the host did about it, when it did something. type CarriedTunnel struct { Interface string `json:"interface"` Port int `json:"port"` Range string `json:"range"` Peers int `json:"peers"` - Taken bool `json:"taken"` + State string `json:"state"` + Note string `json:"note,omitempty"` Kept string `json:"kept,omitempty"` } +// The states a carried tunnel can be in: the found interface still up and the mesh's not; the +// found one down and the mesh's up with its key; or the found one down and the mesh's not up — the +// one state where the peers reach nothing, said as its own word so nothing reads it as either of +// the others. +const ( + CarriedNotTaken = "not-taken" + CarriedTaken = "taken" + CarriedDown = "down" +) + +// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling +// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel +// over, since re-enrolling would rotate every key it holds. Signed with the identity key over +// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. +type Rekey struct { + // Previous is the overlay key this node held until now, as the mesh records it; the mesh + // refuses a rekey naming another, which is how a replayed one is refused. + Previous string `json:"previous"` + OverlayKey string `json:"overlay_key"` + Tunnel *Tunnel `json:"tunnel"` + Proof []byte `json:"proof"` +} + +// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes +// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. +// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). +func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { + var t Tunnel + if tunnel != nil { + t = *tunnel + } + peers := make([]string, 0, len(t.Peers)) + for _, p := range t.Peers { + peers = append(peers, p.PublicKey+"@"+p.Address) + } + return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) +} + // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` diff --git a/internal/link/run.go b/internal/link/run.go index 1379369..2675065 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -377,6 +377,39 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R } // publishReport tells the mesh what this node did, and says whether the broker took it. +// Publish sends one report on this node's own connection and returns: the one-shot path for a +// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same +// account, the same pinned certificate and the same exchange as the running host's reports. +func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error { + config, err := PinnedConfig(m.Fingerprint) + if err != nil { + return err + } + dsn := fmt.Sprintf("amqps://%s:%s@%s/", + url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker) + conn, err := amqp.DialConfig(dsn, amqp.Config{ + TLSClientConfig: config, + Dial: amqp.DefaultDial(timeout), + }) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return err + } + return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err) + } + defer conn.Close() + channel, err := conn.Channel() + if err != nil { + return err + } + defer channel.Close() + var said string + if !publishReport(ctx, channel, m, report, func(s string) { said = s }, timeout) { + return errors.New(said) + } + return nil +} + func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, say Announce, timeout time.Duration) bool { report.Node = m.Node -- 2.54.0