diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index c001e7a..fb8c777 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -140,6 +140,10 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh firewall stay as they are, the foundation's filter is not loaded and the mesh guards its own ports instead, and each module is taken on it one at a time. Without it, a machine in use is refused + --tunnel adopted: the interface of the tunnel the private network takes over + (its key, port, range and peers); found by itself when one is up, and + needed only when several are. --hub-port and --overlay-range then + follow the tunnel The installer carries a builder, not a control plane. What raises a mesh is therefore the same thing that will maintain it, and the control plane a mesh ends up running is @@ -334,6 +338,8 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet { "raise this machine adopted: keep what it runs and its firewall until each module is taken") set.StringVar(&opts.OverlayRange, "overlay-range", opts.OverlayRange, "the private network's address range; must not overlap a tunnel the machine already runs") + set.StringVar(&opts.Tunnel, "tunnel", "", + "adopted: the found tunnel's interface the private network takes over; found by itself when one is up") if opts.Answers == nil { opts.Answers = map[string]string{} } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 681eb3c..b38c6a1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -36,6 +36,7 @@ import ( "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" + "github.com/novox/mesh-host/internal/tunnel" "github.com/novox/mesh-host/internal/upgrade" ) @@ -55,6 +56,8 @@ const usage = `mesh-host — the node host apply FILE make this machine match a declaration from a file reconcile make this machine match the declaration this host carries bundle show what this host carries + overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this + node's overlay key and the mesh is told, signed; --tunnel when several are up owned what this host has applied and still owns version @@ -88,6 +91,7 @@ type options struct { state string token string nodeName string + tunnel string dryRun bool file string } @@ -116,6 +120,8 @@ func parseArgs(args []string) (string, options, error) { set.BoolVar(&opts.dryRun, "dry-run", false, "read and check the declaration, change nothing") set.StringVar(&opts.token, "token", "", "enrol: the one-time token, carried here by a person") set.StringVar(&opts.nodeName, "name", "", "enrol: override the name the token carries") + set.StringVar(&opts.tunnel, "tunnel", "", "enrol, adopted: the found tunnel's interface whose key "+ + "this node takes as its own; found by itself when one is up") // Parsed in a loop, because the standard library stops at the FIRST non-flag argument. // `mesh-host inventory --json` hit that once, and taking the subcommand off the front @@ -143,6 +149,13 @@ func parseArgs(args []string) (string, options, error) { opts.file = positionals[0] return command, opts, nil } + if command == "overlay" { + if len(positionals) != 1 { + return "", opts, errors.New("overlay take [--tunnel ]") + } + opts.file = positionals[0] + return command, opts, nil + } // Anything left over was neither the command nor a flag. Refused rather than ignored: a // mistyped argument that changes nothing and reports success is worse than an error. if len(positionals) > 0 { @@ -230,6 +243,8 @@ func run(ctx context.Context, command string, opts options) error { case "enrol", "enroll": return enrol(ctx, opts) + case "overlay": + return overlayCommand(ctx, opts) case "run": return runLink(ctx, opts) @@ -478,14 +493,39 @@ func enrol(ctx context.Context, opts options) error { } fmt.Printf("generated this node's identity: %s\n", mine.PublicBase64()) - // Its key on the private network, generated here and now for the same reason: the private + // Its key on the private network. Generated here and now, for the same reason: the private // half must never have been anywhere else. The mesh receives only the public half and uses it // to compute a graph it cannot impersonate. - mine.Overlay, err = identity.GenerateOverlayKey() - if err != nil { - return err + // + // **Except on an adopted node with a tunnel** (novox/hq ADR 0105): the found interface's key + // becomes this node's, so the peers that know the tunnel by that key keep reaching it once + // the mesh's interface takes the tunnel over. The one case where the mesh takes a credential + // it did not mint — read from the found configuration, written where a generated one is + // written, never printed, never sent. + var found *link.Tunnel + if token.Adopted { + tun, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) + switch { + case errors.Is(err, tunnel.ErrNone): + fmt.Println("no tunnel is up on this machine; the private network's key is generated") + case err != nil: + return err + default: + mine.Overlay, err = identity.OverlayKeyFrom(tun.PrivateKey()) + if err != nil { + return err + } + found = carried(tun) + fmt.Printf("this node's overlay key is the found tunnel's (%s): %s\n", tun, mine.Overlay.Public) + } + } + if found == nil { + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + return err + } + fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) } - fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) // And the key secrets are sealed to. Here, with the others, because the mesh cannot seal // anything to a key it has not been told about — a key made later would leave a node that @@ -519,7 +559,8 @@ func enrol(ctx context.Context, opts options) error { proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public, sealing.Public, serving.Public)) reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret, - mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, opts.timeout) + mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found, + opts.timeout) if err != nil { return err } @@ -578,6 +619,90 @@ func enrol(ctx context.Context, opts options) error { return nil } +// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over +// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a +// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them +// every credential the mesh sealed to it. +func overlayCommand(ctx context.Context, opts options) error { + if opts.file != "take" { + return errors.New("overlay take [--tunnel ] — the one thing `overlay` does here") + } + identityPath := identity.Path(opts.state) + mine, err := identity.Load(identityPath) + if err != nil { + return err + } + found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) + if err != nil { + return fmt.Errorf("%w. Nothing was changed", err) + } + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + return err + } + fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public) + fmt.Printf(" identity, sealing and serving keys are untouched\n") + + // Told first, then written: a mesh told and a machine not yet written is put right by running + // this again (the mesh refuses the stale second rekey and changes nothing; the files are + // rewritten the same). A machine written and a mesh not told would raise the mesh's interface + // on a key the mesh does not know at the next restart. + if err := link.Publish(ctx, link.Membership{ + Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, + Password: mine.Membership.Password, Signer: mine.Membership.Signer, + }, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil { + return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err) + } + fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node) + + if err := os.WriteFile(identity.OverlayKeyPath(opts.state), + []byte(taken.Overlay.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err) + } + if err := identity.Save(identityPath, taken); err != nil { + return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err) + } + fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n", + identity.OverlayKeyPath(opts.state)) + fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint :%d …`, `plan %s --json`, then push\n", + mine.Node, found.Port, mine.Node) + return nil +} + +// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey +// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same +// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names +// is the one before the take, so the mesh can tell a repeat from a replay. +func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) { + overlay, err := identity.OverlayKeyFrom(found.PrivateKey()) + if err != nil { + return identity.Identity{}, link.Rekey{}, err + } + previous := mine.Overlay.Public + if previous == overlay.Public && mine.OverlayBefore != "" { + previous = mine.OverlayBefore + } + taken := mine + taken.Overlay = overlay + if previous != overlay.Public { + taken.OverlayBefore = previous + } + presented := carried(found) + rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented} + rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented)) + return taken, rekey, nil +} + +// carried is a found tunnel as it is presented to the mesh: everything but its private key. +func carried(t tunnel.Found) *link.Tunnel { + out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, + Address: t.Address, Range: t.Range, PublicKey: t.PublicKey} + for _, p := range t.Peers { + out.Peers = append(out.Peers, link.TunnelPeer{PublicKey: p.PublicKey, Address: p.Address}) + } + return out +} + func firstNonEmpty(values ...string) string { for _, v := range values { if strings.TrimSpace(v) != "" { @@ -891,6 +1016,11 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D if updated.Firewall != nil { report.Firewall = updated.Firewall.Kind } + // And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105). + if t := outcome.Tunnel; t != nil { + report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range, + Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept} + } reached, err := reachable.Collect(ctx, apply.ExecRunner) if err != nil { fmt.Fprintf(os.Stderr, "mesh-host: applied, and could not read what is reachable here: %v\n", err) diff --git a/cmd/mesh-host/rekey_test.go b/cmd/mesh-host/rekey_test.go new file mode 100644 index 0000000..7b19edd --- /dev/null +++ b/cmd/mesh-host/rekey_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/ed25519" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/tunnel" +) + +// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and +// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a +// proof signed by the identity key, over the previous key, the new one and the tunnel. + +func anEnrolledNode(t *testing.T) identity.Identity { + t.Helper() + mine, err := identity.Generate("anchor") + if err != nil { + t.Fatal(err) + } + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa", + Signer: make([]byte, ed25519.PublicKeySize), Password: "p"} + return mine +} + +func aFoundTunnel(t *testing.T) tunnel.Found { + t.Helper() + private, err := identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" + + "Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n")) + if err != nil { + t.Fatal(err) + } + found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf" + return found +} + +func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) { + mine := anEnrolledNode(t) + found := aFoundTunnel(t) + before := mine.Overlay.Public + + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + t.Fatal(err) + } + if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() { + t.Fatal("the overlay key is not the tunnel's") + } + if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) || + taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password || + taken.Membership.Broker != mine.Membership.Broker { + t.Fatal("something other than the overlay key moved") + } + if taken.OverlayBefore != before { + t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore) + } + if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil || + rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 { + t.Fatalf("the rekey does not say what moved: %+v", rekey) + } + if !ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the rekey is not signed by this node's identity key over what it says") + } + if ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the proof is not bound to the node") + } + for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} { + if strings.Contains(said, found.PrivateKey()) { + t.Fatal("the private key travels") + } + } + + // Run again after the take — the mesh not yet told, or told and refused — the previous key it + // names is still the one before the take, so the mesh can tell a repeat from a replay. + again, second, err := rekeyOnto(taken, found) + if err != nil { + t.Fatal(err) + } + if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey { + t.Fatalf("a take run again does not name the key before the first: %+v", second) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index d473f3f..6c4309b 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -57,6 +57,9 @@ type Outcome struct { // Report is what an apply did, in the order it did it. type Report struct { Outcomes []Outcome `json:"outcomes"` + // Tunnel is what this apply says about the tunnel the private network took over, when the + // declaration names one (novox/hq ADR 0105). + Tunnel *TakenTunnel `json:"tunnel,omitempty"` } // Changed reports whether anything about the machine actually moved. An apply that changed @@ -153,6 +156,11 @@ func ApplyKeeping( for _, r := range d.Resources { declared[r.Identity()] = true } + if svc := takesOver(d); svc != nil { + // The found tunnel's configuration is held under an id of its own, declared for as long + // as the service that took it over is (novox/hq ADR 0105). + declared[takeOverID(svc)] = true + } // Which firewall is found here, before anything else, since an unsupported one refuses the // whole declaration (novox/hq ADR 0100). Nothing for a converged node. @@ -327,6 +335,39 @@ func ApplyKeeping( } } + // The private network takes over the tunnel it found, ahead of the service that replaces + // it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never + // flushed. A failure here fails the service too — the mesh's interface is not started on a + // port the found one still holds. + stoppedFound := false + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { + var outcome Outcome + var facts TakenTunnel + var err error + if d.Adoption == nil { + err = errNotAdopted + facts = TakenTunnel{Interface: svc.TakesOver.Interface, State: NotTaken} + } else { + outcome, facts, stoppedFound, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) + } + // Always an account, failure included: the last account standing must never be an + // older "taken" over a machine whose takeover has since gone wrong. + report.Tunnel = &facts + if err != nil { + report.Tunnel.Note = err.Error() + if stoppedFound { + // The found unit is down and the mesh's not up: the one state where the + // peers reach nothing. Started again, and said. + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } + failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report}) + log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) + continue + } + report.Outcomes = append(report.Outcomes, outcome) + log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) + } + was, _ := known.Find(resource.Identity()) var outcome Outcome var err error @@ -346,6 +387,17 @@ func ApplyKeeping( failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { + // The mesh's interface did not come up after the found one was stopped: no + // tunnel at all. The found unit is started again — the machine goes back to + // what it had — and the account says so (novox/hq ADR 0105). + report.Tunnel.Note = "the mesh's interface did not come up: " + err.Error() + if stoppedFound { + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } else { + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + } + } // **A failed action stops what follows. Nothing else does.** // @@ -393,6 +445,11 @@ func ApplyKeeping( known.Release(held.ID) outcome.Detail = takenDetail(held) } + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { + // The found interface is down and the mesh's is up in its place: the tunnel changed + // hands (novox/hq ADR 0105). Read from the machine, not assumed. + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { changed[resource.Identity()] = true diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 0651f4f..be2277c 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -19,6 +19,8 @@ import ( type machine struct { containers map[string]*fakeContainer asked []string + // wgUp is what `wg show interfaces` answers: the tunnels up on the machine. + wgUp string // units are service units by name, as systemd would report them; volumes are the runtime's // named volumes. @@ -29,6 +31,8 @@ type machine struct { type fakeUnit struct { active, enabled string + // wontStart is a unit that accepts `start` and stays inactive — one that starts and dies. + wontStart bool // fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an // administrator installs one. fragment string @@ -63,7 +67,9 @@ func (m *machine) systemctl(args []string) (string, error) { } return u.enabled + "\n", nil case "start": - u.active = "active" + if !u.wontStart { + u.active = "active" + } case "stop": u.active = "inactive" case "enable": @@ -94,6 +100,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e if name == "systemctl" { return m.systemctl(args) } + if name == "wg" { + return m.wgUp, nil + } if name == "getent" { if m.users[args[len(args)-1]] { return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go new file mode 100644 index 0000000..359412c --- /dev/null +++ b/internal/apply/takeover.go @@ -0,0 +1,351 @@ +package apply + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" + "github.com/novox/mesh-host/internal/tunnel" +) + +// The private network takes over the tunnel it found (novox/hq ADR 0105). +// +// The controller says so on the interface's service: `takes-over` names the found interface, the +// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps +// that file like any held file — the original recorded before anything else happens to it — and +// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the +// file is never written, and the found interface goes down the way its own unit takes it down. +// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found +// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands. +// +// Every apply, not once: a found unit somebody starts again would take the port back from the +// mesh's interface, so it is stopped again and said so. That is the one place an adopted node +// undoes something done by hand, and it is because the tunnel is the mesh's now. + +// TakenTunnel is what an apply says about a tunnel it took over, for the node's report. +type TakenTunnel struct { + Interface string + Port int + Range string + Peers int + // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one + // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's + // not up: the peers reach nothing). Note is what this apply did about it. + State string + Note string + Kept string +} + +// The states, as the link says them. +const ( + NotTaken = "not-taken" + Taken = "taken" + TunnelDown = "down" +) + +// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up +// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a +// person takes a moment. Variables so a test need not wait. +var ( + takeoverRecheck = 2 * time.Second + takeoverRechecks = 3 +) + +// takeOverID is the held record's id for the found configuration: the service's own with a suffix, +// so it is declared for as long as the service is and never mistaken for the service itself. +func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" } + +// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that +// replaces it. Returned is the hold's outcome, and what was found for the report. +// +// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's +// option 2 is exactly this going wrong): the declared configuration must listen on the found port +// at the found address, and the key file it points at must hold the found key. Only then is the +// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then +// fails to start can have the found unit started again. +func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration, + known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) { + t := svc.TakesOver + id := takeOverID(svc) + module, _ := d.Adoption.UntakenModuleOf(svc.ID) + if module == "" { + module = "the private network" + } + facts = TakenTunnel{Interface: t.Interface, State: NotTaken} + + // 0. What the found configuration says, before anything: the checks below are against it. + found, ferr := readFoundTunnel(t.Config) + + // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so + // the same code keeps its original, digests it and notices it changing. + file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config} + was, already := known.HeldAt(id) + out, held, err := hold(ctx, sys, file, module, was, already, + "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) + if err != nil { + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + } + known.RecordHeld(held) + facts.Kept = held.Kept + // What the file says, for the report: from the machine, or from the kept original when the + // machine's copy is gone. The private key stays in the file; nothing here keeps it. + unread := "" + if ferr != nil && held.Kept != "" { + found, ferr = readFoundTunnel(held.Kept) + } + if ferr == nil { + facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) + } else { + unread = ferr.Error() + } + + // 2. Where things stand: the found unit, and the mesh's. + foundState, unitErr := sys.ServiceState(ctx, run, t.Unit) + meshState, _ := sys.ServiceState(ctx, run, svc.Unit) + if foundState == "running" && meshState == "running" { + // Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's + // holds — and on a spoke two interfaces with one key flap between them. Not stopped again + // by the mesh: what is found on an adopted node is reported, and the first takeover was + // the one act (the PR note says why). Said, so a person sees it. + facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " + + "`systemctl stop " + t.Unit + "` on the machine" + } + + // 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared + // configuration must listen on the found port at the found address, and the key file it + // points at must hold the found key, or the peers would be dropped the moment it came up. + if foundState == "running" && meshState != "running" { + if ferr != nil { + return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+ + "tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running", + t.Config, ferr, t.Unit) + } + if err := replaces(d, svc, found); err != nil { + return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit) + } + } + + // 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at + // boot. A unit that is not there is not an error — the interface may have been raised + // another way, which the check below catches — and neither is one already down. + var did []string + switch { + case unitErr != nil: + did = append(did, t.Unit+" is not a unit here") + case foundState == "running" && meshState != "running": + if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil { + return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err) + } + after, err := sys.ServiceState(ctx, run, t.Unit) + if err != nil { + return out, facts, true, err + } + if after != "stopped" { + return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) + } + stopped = true + did = append(did, "stopped "+t.Unit) + } + if unitErr == nil { + if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" { + if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil { + return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) + } + did = append(did, "disabled it at boot") + } + } + + // 5. The interface is gone. If it is still up, something other than its unit raised it — + // the predecessor brings its up by hand — and the mesh's interface cannot take its port + // and address while it is. Looked at again for a moment, since a person taking it down + // takes a moment; then refused, naming what to do. + if meshState != "running" { + for try := 0; ; try++ { + if !interfaceUp(ctx, run, t.Interface) { + break + } + if try >= takeoverRechecks { + return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+ + "raised by hand, not by its unit, and the mesh's interface cannot take its port and "+ + "address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+ + "over. Nothing was flushed", t.Interface, t.Unit, t.Interface) + } + select { + case <-ctx.Done(): + return out, facts, stopped, ctx.Err() + case <-time.After(takeoverRecheck): + } + } + } + + out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found" + if held.Kept != "" { + out.Detail += " (original at " + held.Kept + ")" + } + if len(did) > 0 { + out.Detail += "; " + strings.Join(did, ", ") + " — never flushed" + } + if held.Changed != "" { + out.Detail += "; " + held.Changed + " by something other than the mesh since it was found" + } + if unread != "" { + // Said, not swallowed: the report would otherwise say a tunnel with no port and no + // peers was carried, which reads as a tunnel that was not one. + out.Detail += "; what it says could not be read as a tunnel's: " + unread + } + return out, facts, stopped, nil +} + +// readFoundTunnel is the found configuration as a tunnel. +func readFoundTunnel(path string) (tunnel.Found, error) { + raw, err := os.ReadFile(path) + if err != nil { + return tunnel.Found{}, err + } + return tunnel.Parse(raw) +} + +// interfaceUp is whether a WireGuard interface is up on the machine. +func interfaceUp(ctx context.Context, run Runner, iface string) bool { + up, err := run(ctx, "wg", "show", "interfaces") + if err != nil { + return false + } + for _, name := range strings.Fields(up) { + if name == iface { + return true + } + } + return false +} + +// replaces holds the mesh's declared interface configuration against the found tunnel it is to +// replace: same port, same address, and a key file holding the found key. The configuration is +// the file the service restarts on; its `PostUp = wg set %i private-key ` names the key. +func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error { + var conf *declaration.File + for _, r := range d.Resources { + f, ok := r.(*declaration.File) + if !ok { + continue + } + for _, id := range svc.RestartOn { + if f.ID == id { + conf = f + } + } + } + if conf == nil { + return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+ + "raise cannot be checked against the found one", svc.Unit, found.Interface) + } + port, address, keyPath := "", "", "" + for _, line := range strings.Split(conf.Content, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + switch key { + case "listenport": + port = value + case "address": + address = strings.TrimSpace(strings.Split(value, ",")[0]) + case "postup": + if _, after, ok := strings.Cut(value, "private-key "); ok { + keyPath = strings.Fields(after)[0] + } + } + } + var wrong []string + if port != fmt.Sprint(found.Port) { + wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port)) + } + if host(address) != host(found.Address) { + wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address)) + } + switch raw, err := os.ReadFile(keyPath); { + case keyPath == "": + wrong = append(wrong, "it names no key file") + case err != nil: + wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err)) + default: + public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw))) + if perr != nil || public != found.PublicKey { + wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+ + "--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface)) + } + } + if len(wrong) > 0 { + return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+ + "dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again", + found.Interface, strings.Join(wrong, "; ")) + } + return nil +} + +// host is an address without its prefix length. +func host(address string) string { + if i := strings.Index(address, "/"); i >= 0 { + return address[:i] + } + return address +} + +// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up +// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is +// down — the peers reach nothing. +func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string { + foundState, _ := sys.ServiceState(ctx, run, foundUnit) + meshState, _ := sys.ServiceState(ctx, run, meshUnit) + foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@")) + switch { + case meshState == "running" && !foundUp: + return Taken + case meshState == "running": + // Both up: not a takeover that holds, and said as not taken so nobody reads it as one. + return NotTaken + case foundUp: + return NotTaken + default: + return TunnelDown + } +} + +// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the +// machine has the tunnel it had rather than none, and says so in the account. +func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) { + if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil { + facts.State = TunnelDown + facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit + return + } + if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" { + facts.State = TunnelDown + facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit + return + } + facts.State = NotTaken + facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" +} + +// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since +// a machine has one private network. +func takesOver(d *declaration.Declaration) *declaration.Service { + for _, r := range d.Resources { + if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil { + return svc + } + } + return nil +} + +// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the +// parser refuses already; kept as a second line of defence at the point of acting. +var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only") diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go new file mode 100644 index 0000000..18fde16 --- /dev/null +++ b/internal/apply/takeover_test.go @@ -0,0 +1,256 @@ +package apply + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the +// found interface without flushing it, and keeps its configuration — and stops nothing until the +// mesh's interface is known to be able to replace it. + +// foundKey is the predecessor's private key, a real one made once per run: the key is what the +// takeover must never print or copy, so it had better be one. +var foundKey = func() string { + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + panic(err) + } + return base64.StdEncoding.EncodeToString(k.Bytes()) +}() + +var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" + + "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + + "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" + +// aTakeover is the private network's declaration for an adopted hub whose interface takes over +// the found tunnel: the mesh's configuration — on the found port and address, its key set from the +// node's own key file, the found peers in its list — and the interface's service naming what it +// replaces. Port and address are parameters so a test can declare a wrong one. +func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration { + t.Helper() + return adopted(t, + `{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`, + `{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600", + "content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, + {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled", + "restart-on":["mesh-wireguard.overlay-config"], + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`) +} + +// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found +// configuration on disk, and the node's key file holding the found key, as enrolment left it. +func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) { + t.Helper() + dir = t.TempDir() + config = filepath.Join(dir, "wg0.conf") + mesh = filepath.Join(dir, "mesh0.conf") + keyFile = filepath.Join(dir, "overlay.key") + if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil { + t.Fatal(err) + } + m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{ + "wg-quick@wg0": {active: "active", enabled: "enabled"}, + "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, + }} + takeoverRecheck = 0 + return dir, config, mesh, keyFile, m +} + +func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) + + // The found interface: its unit stopped and disabled, and nothing else done to it. + if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { + t.Fatalf("the found unit was not stopped and disabled: %+v", u) + } + for _, asked := range m.asked { + if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") { + t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked) + } + if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") { + t.Errorf("the found interface was flushed: %q", asked) + } + } + // Its configuration: on disk as it was, its original kept, held for the module. + if got, _ := os.ReadFile(config); string(got) != foundConf { + t.Fatalf("the found configuration was changed:\n%s", got) + } + held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over") + if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" { + t.Fatalf("the found configuration is not held: %+v", held) + } + if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf { + t.Fatalf("the original was not kept as found: %q", kept) + } + // The mesh's interface: up, enabled, with the found peers in the file the mesh wrote. + if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" { + t.Fatalf("the mesh's interface was not raised: %+v", u) + } + if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") { + t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got) + } + // And the report says so, with what was found — port, range, peers — and never the key. + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 || + report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept { + t.Fatalf("the report does not say what was carried: %+v", report.Tunnel) + } + for _, o := range report.Outcomes { + if strings.Contains(o.Detail, foundKey) { + t.Errorf("the found key was printed in an outcome: %+v", o) + } + } + if strings.Contains(report.Tunnel.Note, foundKey) { + t.Error("the found key was printed in the account") + } + if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" || + !strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") { + t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o) + } + if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded { + t.Error("the found configuration was recorded as applied, so it would be removed as an orphan") + } +} + +func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + otherKey := filepath.Join(dir, "other.key") + k, _ := ecdh.X25519().GenerateKey(rand.Reader) + if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cases := map[string]*declaration.Declaration{ + "another port": aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"), + "another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"), + "another key": aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"), + "no key file": aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"), + } + for name, d := range cases { + m.asked = nil + report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{}, + store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") { + t.Fatalf("%s: the takeover was not refused: %v", name, err) + } + if name == "another key" && !strings.Contains(err.Error(), "overlay take") { + t.Errorf("%s: the refusal does not name the remedy: %v", name, err) + } + if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") { + t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name) + } + if m.units["wg-quick@mesh0"].active == "active" { + t.Fatalf("%s: the mesh's interface was started on top of the found one", name) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") { + t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel) + } + if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { + t.Errorf("%s: the found configuration was not kept before the refusal", name) + } + } +} + +func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.units["wg-quick@mesh0"].wontStart = true + report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil { + t.Fatal("a mesh interface that did not come up was reported as applied") + } + if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") { + t.Fatalf("the found unit was not stopped and then started again: %v", m.asked) + } + if m.units["wg-quick@wg0"].active != "active" { + t.Fatal("the machine was left with no tunnel at all") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || + !strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") { + t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel) + } +} + +func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") + _, state := applyAdopted(t, d, store.State{}, m, dir) + m.asked = nil + + report, again := applyAdopted(t, d, state, m, dir) + if report.Changed() { + t.Errorf("a second apply moved the machine: %+v", report.Outcomes) + } + if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still { + t.Error("the hold on the found configuration was forgotten while the service still declares it") + } + if m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit already down was stopped again") + } + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel) + } + + // Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh — + // on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said. + m.units["wg-quick@wg0"].active = "active" + m.asked = nil + report, _ = applyAdopted(t, d, again, m, dir) + if m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit started again by hand was stopped by the mesh") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") { + t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel) + } +} + +func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + // The unit is not running, yet the interface is up: the predecessor raised it by hand. + m.units["wg-quick@wg0"].active = "inactive" + m.wgUp = "wg0 mesh0\n" + report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") { + t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err) + } + if m.units["wg-quick@mesh0"].active == "active" { + t.Error("the mesh's interface was started on a port the found one still holds") + } + // Looked at more than once before giving up: a person taking it down takes a moment. + shows := 0 + for _, a := range m.asked { + if a == "wg show interfaces" { + shows++ + } + } + if shows < takeoverRechecks+1 { + t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken { + t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel) + } + if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { + t.Error("the found configuration was not kept before the refusal") + } +} + +func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) { + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`)) + if err == nil || !strings.Contains(err.Error(), "adopted") { + t.Fatalf("a takeover on a converged node was accepted: %v", err) + } +} diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 4349bcc..12089b0 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -36,6 +36,7 @@ import ( "time" "github.com/novox/mesh-host/internal/firewall" + "github.com/novox/mesh-host/internal/tunnel" ) // Step names one stage. A failure says which one, because "the bootstrap failed" is a sentence @@ -201,6 +202,11 @@ type Options struct { // until each module is taken, its firewall stays in force, and the mesh guards its own ports // in a table that only refuses. Without it, a machine in use is refused. Adopted bool + // Tunnel names the found tunnel's interface an adopted hub takes over (novox/hq ADR 0105), when + // more than one is up and the machine cannot say which. Empty finds the one that is up. Once + // found, the tunnel's port is the hub's and its range the private network's; --hub-port and + // --overlay-range may agree with it or be left unsaid. + Tunnel string } // pivots reports whether this run goes past the foundation. @@ -311,6 +317,9 @@ type Result struct { // Filter is the packet filter chosen for when the node converges; an adopted genesis loads // none, and the flip assigns this one. Filter string `json:"filter-on-converge,omitempty"` + // Tunnel is the found tunnel an adopted genesis takes over (novox/hq ADR 0105): what was read + // from it, never its key. + Tunnel *tunnel.Found `json:"tunnel,omitempty"` } // Run performs the bootstrap, saying what it is doing as it goes. @@ -394,6 +403,23 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro } result.Firewall = string(kind) say(" adopted what is on this machine is kept; its firewall (" + string(kind) + ") stays in force") + + // The tunnel the predecessor left, which the private network takes over (novox/hq ADR + // 0105): its port is the hub's and its range is the mesh's from here on, so both are + // settled before the ports are checked free and the bundle rewritten. + found, err := TakeTheTunnel(&o, d.Run) + if err != nil { + return result, failed(StepPreflight, err) + } + if found != nil { + result.Tunnel = found + result.Ports = o.Ports + say(fmt.Sprintf(" tunnel %s — the private network takes it over: its port %d is "+ + "the hub's, its range %s the mesh's, and its %d peer(s) are carried until they enrol", + found.Interface, found.Port, found.Range, len(found.Peers))) + } else { + say(" tunnel none up on this machine; the private network is raised on its own port and range") + } } sys, err := WorkOutSystem(ctx, d.Run, o.System) diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index 7ee3b71..0137c3b 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -112,7 +112,14 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla return out, err } joining, cancel := context.WithTimeout(ctx, o.Wait) - joined, err := control.run(joining, o.Host, "enrol", "--token", token, "--state", o.State) + args := []string{"enrol", "--token", token, "--state", o.State} + if o.Tunnel != "" { + // The found tunnel's key becomes this node's overlay key, and the tunnel travels with + // the enrolment (novox/hq ADR 0105). Named, so the host takes the one genesis settled + // its ports and range on and not another that came up since. + args = append(args, "--tunnel", o.Tunnel) + } + joined, err := control.run(joining, o.Host, args...) cancel() if err != nil { return out, fmt.Errorf( diff --git a/internal/bootstrap/ports.go b/internal/bootstrap/ports.go index c3c1ed8..cb09372 100644 --- a/internal/bootstrap/ports.go +++ b/internal/bootstrap/ports.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net" "sort" @@ -13,6 +14,7 @@ import ( "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/tunnel" ) // FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100). @@ -321,6 +323,36 @@ func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(rea return nil } +// TakeTheTunnel finds the tunnel an adopted machine's private network takes over (novox/hq ADR +// 0105) and settles the options on it: the hub's port is the tunnel's, the mesh's range is the +// tunnel's, and the interface is named for the enrolment that takes its key. Nil when no tunnel is +// up, which is an ordinary machine. A --hub-port or --overlay-range that disagrees with the +// tunnel is refused: the peers dial the tunnel's port and live in its range, and a mesh raised +// beside them on other numbers is the two-tunnel shape the record rejects. +func TakeTheTunnel(o *Options, run Runner) (*tunnel.Found, error) { + found, err := tunnel.Find(context.Background(), tunnel.Runner(run), o.Tunnel) + if errors.Is(err, tunnel.ErrNone) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("%w. An adopted hub takes over the tunnel it finds; nothing was changed", err) + } + if o.Ports.Hub != 0 && o.Ports.Hub != DefaultPorts().Hub && o.Ports.Hub != found.Port { + return nil, fmt.Errorf("--hub-port %d disagrees with the tunnel %s, which listens on %d: the "+ + "private network takes over that tunnel on its own port, so leave --hub-port unsaid or "+ + "say %d", o.Ports.Hub, found.Interface, found.Port, found.Port) + } + if o.OverlayRange != "" && o.OverlayRange != DefaultOverlayRange && o.OverlayRange != found.Range { + return nil, fmt.Errorf("--overlay-range %s disagrees with the tunnel %s, whose range is %s: the "+ + "private network takes over that tunnel with its range, so leave --overlay-range unsaid "+ + "or say %s", o.OverlayRange, found.Interface, found.Range, found.Range) + } + o.Tunnel = found.Interface + o.Ports.Hub = found.Port + o.OverlayRange = found.Range + return &found, nil +} + // OverlayClear refuses a private-network range that overlaps an address or a route the machine // already has — a predecessor's tunnel still running — naming the interface. The mesh's own // interface is not counted. @@ -459,12 +491,26 @@ func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declara } return false } + if o.Tunnel != "" { + // The hub's port is the found tunnel's, held by that tunnel until the mesh's interface + // takes it over (novox/hq ADR 0105): held by design, not by something else. + inner := ours + ours = func(r reachable.Reach) bool { + return inner(r) || (r.Protocol == "udp" && r.Port == p.Hub) + } + } if err := PortsFree(ctx, run, p, ours); err != nil { return err } say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp", p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub)) - if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { + if o.Tunnel != "" { + // One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the + // two must not overlap applies only where a found tunnel is left running beside the mesh's + // (ADR 0100, narrowed by ADR 0105). + say(fmt.Sprintf(" range %s is the tunnel %s's, taken over; not checked against it", + o.OverlayRange, o.Tunnel)) + } else if err := OverlayClear(ctx, run, o.OverlayRange); err != nil { return err } if err := NamesFree(ctx, run, names, known); err != nil { diff --git a/internal/bootstrap/ports_test.go b/internal/bootstrap/ports_test.go index 366a3dd..ac0ec92 100644 --- a/internal/bootstrap/ports_test.go +++ b/internal/bootstrap/ports_test.go @@ -2,6 +2,9 @@ package bootstrap import ( "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" "errors" "os" "path/filepath" @@ -12,6 +15,7 @@ import ( "github.com/novox/mesh-host/internal/declaration" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/tunnel" ) // Defends novox/hq ADR 0100: the foundation's ports are the node's — inputs to genesis, checked free, @@ -130,9 +134,9 @@ func TestTwoThingsOnOnePortAreRefused(t *testing.T) { // machineRunner answers ss, docker ps, docker inspect and ip from fixtures. type machineRunner struct { - ss, ps, addrs, routes string - unlabelled map[string]bool - labelled map[string]bool + ss, ps, addrs, routes, wg string + unlabelled map[string]bool + labelled map[string]bool } func (m machineRunner) run(_ context.Context, name string, args ...string) (string, error) { @@ -154,6 +158,8 @@ func (m machineRunner) run(_ context.Context, name string, args ...string) (stri return m.addrs, nil case name == "ip" && args[1] == "route": return m.routes, nil + case name == "wg": + return m.wg, nil } return "", nil } @@ -295,3 +301,76 @@ func TestARerunOfGenesisFindsItsOwnPackageRegistry(t *testing.T) { t.Error("a container under the package registry's name on a fresh machine was not refused") } } + +// novox/hq ADR 0105: an adopted genesis takes over the tunnel it finds — its port is the hub's, +// its range the mesh's, and neither is refused for being held by it. +func TestAnAdoptedGenesisSettlesOnTheTunnelItFinds(t *testing.T) { + private, err := aFoundKey() + if err != nil { + t.Fatal(err) + } + conf := "[Interface]\nPrivateKey = " + private + "\nListenPort = 51900\nAddress = 192.0.2.1/24\n" + + "[Peer]\nPublicKey = PEER=\nAllowedIPs = 192.0.2.2/32\n" + tunnel.ReadFile = func(path string) ([]byte, error) { + if path == tunnel.ConfigDir+"/wg0.conf" { + return []byte(conf), nil + } + return nil, errors.New("no such file") + } + t.Cleanup(func() { tunnel.ReadFile = os.ReadFile }) + m := machineRunner{ + wg: "wg0\n", + ss: "udp UNCONN 0 0 0.0.0.0:51900 0.0.0.0:*\n", + addrs: "5: wg0 inet 192.0.2.1/24 scope global wg0\n", + routes: "192.0.2.0/24 dev wg0 proto kernel scope link src 192.0.2.1\n", + } + + o := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange, State: filepath.Join(t.TempDir(), "state.json")} + found, err := TakeTheTunnel(&o, m.run) + if err != nil || found == nil { + t.Fatalf("the tunnel was not found and taken: %+v %v", found, err) + } + if o.Tunnel != "wg0" || o.Ports.Hub != 51900 || o.OverlayRange != "192.0.2.0/24" { + t.Fatalf("genesis did not settle on the tunnel's port and range: %+v", o) + } + // Its port is held by the tunnel and its range overlaps the tunnel's — by design, not refused. + if err := CheckTheMachine(context.Background(), o, m.run, producedBundle(t).Declaration, func(string) {}); err != nil { + t.Fatalf("the machine was refused for the tunnel it takes over: %v", err) + } + // Whereas the same machine not taking it over is refused on both counts (ADR 0100). + plain := o + plain.Tunnel = "" + if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil || + !strings.Contains(err.Error(), "51900") { + t.Fatalf("a tunnel not taken over stopped being refused for holding the hub's port: %v", err) + } + plain.Ports.Hub = 51821 + if err := CheckTheMachine(context.Background(), plain, m.run, producedBundle(t).Declaration, func(string) {}); err == nil || + !strings.Contains(err.Error(), "wg0") { + t.Fatalf("a tunnel not taken over stopped being refused for overlapping the range: %v", err) + } + + // Numbers that disagree with the tunnel are refused, naming the tunnel's. + for name, given := range map[string]Options{ + "--hub-port": {Adopted: true, Ports: FoundationPorts{Hub: 51821}, OverlayRange: DefaultOverlayRange}, + "--overlay-range": {Adopted: true, Ports: DefaultPorts(), OverlayRange: "10.77.0.0/16"}, + } { + if _, err := TakeTheTunnel(&given, m.run); err == nil || !strings.Contains(err.Error(), name) { + t.Errorf("a %s disagreeing with the tunnel was accepted: %v", name, err) + } + } + // And no tunnel up is an ordinary machine. + m.wg = "mesh0\n" + none := Options{Adopted: true, Ports: DefaultPorts(), OverlayRange: DefaultOverlayRange} + if found, err := TakeTheTunnel(&none, m.run); err != nil || found != nil || none.Ports.Hub != DefaultPorts().Hub { + t.Errorf("a machine with no tunnel was not left as it was: %+v %v", found, err) + } +} + +func aFoundKey() (string, error) { + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(k.Bytes()), nil +} diff --git a/internal/declaration/adoption_test.go b/internal/declaration/adoption_test.go index 1ca0d3b..e422516 100644 --- a/internal/declaration/adoption_test.go +++ b/internal/declaration/adoption_test.go @@ -105,3 +105,28 @@ func TestACarriedBundleCannotSayTheNodeIsAdopted(t *testing.T) { t.Fatalf("a bundle claiming adoption was not refused: %v", err) } } + +// novox/hq ADR 0105: a service may take over a found tunnel, said whole and on an adopted node. +func TestTakingOverATunnelIsSaidWholeAndForARunningService(t *testing.T) { + adoptedWith := func(service string) error { + _, err := Parse([]byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[` + service + `]}`)) + return err + } + good := `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}` + if err := adoptedWith(good); err != nil { + t.Fatalf("a whole takeover on an adopted node was refused: %v", err) + } + for name, bad := range map[string]string{ + "its own unit": `{"id":"up","type":"service","unit":"wg-quick@wg0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + "no config": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0"}}`, + "a stopped service": `{"id":"up","type":"service","unit":"wg-quick@mesh0","state":"stopped", + "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}`, + } { + if err := adoptedWith(bad); err == nil { + t.Errorf("a takeover naming %s was accepted", name) + } + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index c0bc7af..237d6ae 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -617,6 +617,21 @@ type Service struct { // container runtime, whose restart stops every container on the machine (novox/hq ADR 0102). // A change that is also in RestartOn restarts it, which covers a reload. ReloadOn []string `json:"reload-on,omitempty"` + + // TakesOver names the found tunnel this service replaces (novox/hq ADR 0105): before this unit + // is started, the named unit is stopped and disabled — never flushed — and its configuration + // file is kept like any held file. Only on an adopted node, and only said by the controller, + // which knows the found tunnel's key is this node's own: without that, starting this unit on + // the found one's port would drop every peer's packets. + TakesOver *TakeOver `json:"takes-over,omitempty"` +} + +// TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its +// configuration file. +type TakeOver struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` } func (s *Service) Identity() string { return s.ID } @@ -637,6 +652,19 @@ func (s *Service) validate(where string, _ bool) []string { "%s: boot %q; a service is \"enabled\" or \"disabled\" at boot, or omits it to "+ "leave the machine's own setting alone", where, s.Boot)) } + if t := s.TakesOver; t != nil { + switch { + case t.Unit == "" || t.Config == "" || t.Interface == "": + problems = append(problems, where+": takes-over names the found tunnel's interface, unit "+ + "and config, and this leaves one out") + case t.Unit == s.Unit: + problems = append(problems, fmt.Sprintf("%s: takes-over names %s, which is this service's own unit", + where, t.Unit)) + case s.State != "running": + problems = append(problems, where+": a service that takes over a tunnel is running — stopping "+ + "the found one for a service that will not run would leave the peers with nothing") + } + } return problems } @@ -1167,6 +1195,14 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { "resource %q: an opening is for an adopted node, and this declaration does not "+ "say the node is adopted", r.Identity())) } + if svc, ok := r.(*Service); ok && svc.TakesOver != nil { + // A tunnel is taken over on an adopted node, where what is found is kept: on a + // converged one there is nothing found to take over, and stopping a unit the + // mesh did not declare would be the host deciding (novox/hq ADR 0105). + problems = append(problems, fmt.Sprintf( + "resource %q: taking over a tunnel is for an adopted node, and this declaration "+ + "does not say the node is adopted", r.Identity())) + } } } diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 9ff2ee8..c8d7430 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -49,8 +49,13 @@ type Identity struct { Membership Membership `json:"membership"` // Overlay is this node's key on the private network. Generated here, like the identity above, - // and for the same reason: the mesh computes a graph it cannot impersonate. + // and for the same reason: the mesh computes a graph it cannot impersonate — or, on an adopted + // node that took a found tunnel over, that tunnel's key (novox/hq ADR 0105). Overlay OverlayKey `json:"overlay"` + // OverlayBefore is the public half of the overlay key this node held before it took a found + // tunnel's, so a take run again names the key the mesh still records. Empty on a node that + // never took one. + OverlayBefore string `json:"overlay_before,omitempty"` } // Membership is how this node reaches the mesh it belongs to, and who it believes. diff --git a/internal/identity/overlay.go b/internal/identity/overlay.go index 4c3eb39..bc1342c 100644 --- a/internal/identity/overlay.go +++ b/internal/identity/overlay.go @@ -43,6 +43,27 @@ func GenerateOverlayKey() (OverlayKey, error) { }, nil } +// OverlayKeyFrom makes this node's overlay key from a private key it did not generate: the found +// tunnel's, on an adopted node whose private network takes that tunnel over (novox/hq ADR 0105). +// The one case where the mesh takes a credential it did not mint. From here on it is stored and +// sealed exactly as a generated one — in the identity file and the key file, readable by root +// alone — and the mesh receives only the public half, derived here from the private one so the +// two cannot disagree. +func OverlayKeyFrom(privateBase64 string) (OverlayKey, error) { + raw, err := base64.StdEncoding.DecodeString(privateBase64) + if err != nil { + return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not base64: %w", err) + } + private, err := ecdh.X25519().NewPrivateKey(raw) + if err != nil { + return OverlayKey{}, fmt.Errorf("the found tunnel's private key is not a Curve25519 key: %w", err) + } + return OverlayKey{ + Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(private.Bytes()), + }, nil +} + // OverlayKeyPath is where the private half lives: a file of its own, referenced by the interface // configuration rather than embedded in it. // diff --git a/internal/identity/overlay_test.go b/internal/identity/overlay_test.go new file mode 100644 index 0000000..9ce8a76 --- /dev/null +++ b/internal/identity/overlay_test.go @@ -0,0 +1,28 @@ +package identity + +import ( + "strings" + "testing" +) + +// novox/hq ADR 0105: the found tunnel's private key becomes the node's overlay key, stored as a +// generated one is, and the public half the mesh records is derived from it — so the peers that +// know the tunnel by that key keep reaching it. +func TestAnOverlayKeyTakenFromAFoundTunnelIsTheSameKey(t *testing.T) { + generated, err := GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + taken, err := OverlayKeyFrom(generated.Private) + if err != nil { + t.Fatal(err) + } + if taken.Public != generated.Public || taken.Private != generated.Private { + t.Fatalf("a key taken from a private half is not that key: %+v vs %+v", taken, generated) + } + for _, bad := range []string{"", "not base64!", "c2hvcnQ="} { + if _, err := OverlayKeyFrom(bad); err == nil || !strings.Contains(err.Error(), "found tunnel") { + t.Errorf("%q was taken as a key: %v", bad, err) + } + } +} diff --git a/internal/link/enrol.go b/internal/link/enrol.go index 6cf839b..8f755c8 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -52,6 +52,30 @@ type EnrolRequest struct { // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish // on a token this key already spent (novox/hq issue 083). Proof []byte `json:"proof,omitempty"` + + // Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR + // 0105): everything about it but that key. Sent with the keys because it is one of them — + // OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the + // hub's address, the range and the carried peers from it before the first declaration. + Tunnel *Tunnel `json:"tunnel,omitempty"` +} + +// Tunnel is a found tunnel as it travels: no private key. +type Tunnel struct { + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + Port int `json:"port"` + Address string `json:"address"` + Range string `json:"range"` + PublicKey string `json:"public_key"` + Peers []TunnelPeer `json:"peers,omitempty"` +} + +// TunnelPeer is one peer of a found tunnel: its key, and the address the tunnel routes to it. +type TunnelPeer struct { + PublicKey string `json:"public_key"` + Address string `json:"address"` } // EnrolReply is what the mesh says back. @@ -125,7 +149,7 @@ func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { // the broker who connected. func Enrol(ctx context.Context, address, pin, node, secret string, public []byte, overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, - timeout time.Duration) (EnrolReply, error) { + tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) { config, err := PinnedConfig(pin) if err != nil { @@ -172,7 +196,7 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, - Proof: proof} + Proof: proof, Tunnel: tunnel} body, err := json.Marshal(request) if err != nil { return EnrolReply{}, err diff --git a/internal/link/messages.go b/internal/link/messages.go index 953fe6c..76765a0 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,6 +1,10 @@ package link -import "time" +import ( + "strconv" + "strings" + "time" +) // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field @@ -100,6 +104,67 @@ type Report struct { // published container port. Only an adopted node reports it; it is what converging the node // previews, so nothing closes without being named first. Reachable []Reach `json:"reachable,omitempty"` + + // Tunnel is what this adopted node says about the tunnel it found and carried (novox/hq ADR + // 0105): which interface, its port, range and peer count, whether the found interface is down + // and the mesh's up in its place, and where the found configuration's original was kept. + Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + + // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq + // ADR 0105). Not an account of the machine: a report carrying one says nothing else. + Rekey *Rekey `json:"rekey,omitempty"` +} + +// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried +// states below; Note is what the host did about it, when it did something. +type CarriedTunnel struct { + Interface string `json:"interface"` + Port int `json:"port"` + Range string `json:"range"` + Peers int `json:"peers"` + State string `json:"state"` + Note string `json:"note,omitempty"` + Kept string `json:"kept,omitempty"` +} + +// The states a carried tunnel can be in: the found interface still up and the mesh's not; the +// found one down and the mesh's up with its key; or the found one down and the mesh's not up — the +// one state where the peers reach nothing, said as its own word so nothing reads it as either of +// the others. +const ( + CarriedNotTaken = "not-taken" + CarriedTaken = "taken" + CarriedDown = "down" +) + +// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling +// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel +// over, since re-enrolling would rotate every key it holds. Signed with the identity key over +// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. +type Rekey struct { + // Previous is the overlay key this node held until now, as the mesh records it; the mesh + // refuses a rekey naming another, which is how a replayed one is refused. + Previous string `json:"previous"` + OverlayKey string `json:"overlay_key"` + Tunnel *Tunnel `json:"tunnel"` + Proof []byte `json:"proof"` +} + +// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes +// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. +// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). +func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { + var t Tunnel + if tunnel != nil { + t = *tunnel + } + peers := make([]string, 0, len(t.Peers)) + for _, p := range t.Peers { + peers = append(peers, p.PublicKey+"@"+p.Address) + } + return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) } // Held is one file or container found on an adopted node and kept as it was. diff --git a/internal/link/run.go b/internal/link/run.go index 1379369..2675065 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -377,6 +377,39 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R } // publishReport tells the mesh what this node did, and says whether the broker took it. +// Publish sends one report on this node's own connection and returns: the one-shot path for a +// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same +// account, the same pinned certificate and the same exchange as the running host's reports. +func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error { + config, err := PinnedConfig(m.Fingerprint) + if err != nil { + return err + } + dsn := fmt.Sprintf("amqps://%s:%s@%s/", + url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker) + conn, err := amqp.DialConfig(dsn, amqp.Config{ + TLSClientConfig: config, + Dial: amqp.DefaultDial(timeout), + }) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return err + } + return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err) + } + defer conn.Close() + channel, err := conn.Channel() + if err != nil { + return err + } + defer channel.Close() + var said string + if !publishReport(ctx, channel, m, report, func(s string) { said = s }, timeout) { + return errors.New(said) + } + return nil +} + func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, say Announce, timeout time.Duration) bool { report.Node = m.Node diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go new file mode 100644 index 0000000..53c6bce --- /dev/null +++ b/internal/tunnel/tunnel.go @@ -0,0 +1,274 @@ +// Package tunnel reads the tunnel a predecessor left on a machine, so the mesh's private network +// can take it over in place (novox/hq ADR 0105). +// +// On an adopted node that is the hub, the mesh's interface is raised with the found interface's +// private key, on its port, with its address and range, and every peer it had. The found interface +// is stopped, never flushed; its configuration stays on disk. What this package does is the +// reading: which interface is there, what its file says, and what of that travels to the mesh — +// everything but the private key, which becomes the node's own overlay key and is stored the way +// that key is stored. +package tunnel + +import ( + "context" + "crypto/ecdh" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "sort" + "strconv" + "strings" +) + +// Runner executes a command. The same shape as everywhere else in this host. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// MeshInterface is the private network's own interface, which is never the found one. +const MeshInterface = "mesh0" + +// ConfigDir is where wg-quick keeps an interface's configuration. +const ConfigDir = "/etc/wireguard" + +// Found is a tunnel as found on the machine: everything the mesh is told about it, and the +// private key, which it is not. +type Found struct { + // Interface, Unit and Config are what the mesh's interface takes over. + Interface string `json:"interface"` + Unit string `json:"unit"` + Config string `json:"config"` + // Port is the port the interface listens on; Address its own address with prefix length; + // Range the network that prefix names. + Port int `json:"port"` + Address string `json:"address"` + Range string `json:"range"` + // PublicKey is what every peer knows this tunnel by — derived here from the private key, so + // it is the key the file actually holds and not a comment beside it. + PublicKey string `json:"public_key"` + Peers []Peer `json:"peers,omitempty"` + + // privateKey never travels and never prints: not in JSON, not in %v. It is read once, to + // become the node's overlay key, and the file it came from is kept as found. + privateKey string +} + +// Peer is one peer of the found tunnel. +type Peer struct { + PublicKey string `json:"public_key"` + // Address is the one address the tunnel routes to the peer, as the file's AllowedIPs said it + // (with or without a /32). + Address string `json:"address"` + // Endpoint is where the found tunnel dialled the peer, if it did. Not carried to the mesh — + // a carried peer dials in, as it always did — but kept so a person reading the report sees + // what the file said. + Endpoint string `json:"endpoint,omitempty"` +} + +// PrivateKey is the found interface's private key, base64 as WireGuard writes it. The one +// accessor; a caller that has it is taking it as the node's key. +func (f Found) PrivateKey() string { return f.privateKey } + +// String is what a found tunnel prints as: never the key. +func (f Found) String() string { + return fmt.Sprintf("%s on port %d, %s in %s, %d peer(s)", f.Interface, f.Port, f.Address, + f.Range, len(f.Peers)) +} + +// MarshalJSON writes everything but the private key, whatever a caller passes to an encoder. +func (f Found) MarshalJSON() ([]byte, error) { + type wire Found + return json.Marshal(wire(f)) +} + +// ErrNone is a machine with no tunnel to take over. +var ErrNone = errors.New("no tunnel is up on this machine besides the mesh's own") + +// ErrSeveral is a machine with more than one, when nobody said which. +var ErrSeveral = errors.New("more than one tunnel is up on this machine") + +// ReadFile is how a configuration is read; a variable so a test can hand in a file. +var ReadFile = os.ReadFile + +// Find reads the tunnel to take over: the one named, or the one interface up besides the mesh's +// own. Nothing up is ErrNone — an ordinary answer, the machine has no tunnel to adopt — and two +// or more with none named is ErrSeveral, naming them, because choosing would be deciding. +// +// Read from the interface's configuration file rather than from the running interface: the file +// is what wg-quick raised and what carries the address, which the kernel does not report per +// interface the way the key and peers are. The running interface is consulted only to know the +// tunnel is up — a file for an interface nothing runs is not a tunnel the peers are reaching. +func Find(ctx context.Context, run Runner, named string) (Found, error) { + out, err := run(ctx, "wg", "show", "interfaces") + if err != nil { + return Found{}, fmt.Errorf("cannot ask which tunnels are up on this machine: %w", err) + } + var up []string + for _, iface := range strings.Fields(out) { + if iface != MeshInterface { + up = append(up, iface) + } + } + sort.Strings(up) + iface := named + switch { + case named != "": + found := false + for _, u := range up { + if u == named { + found = true + } + } + if !found { + return Found{}, fmt.Errorf("%s was named as the tunnel to take over and is not up; up: %s", + named, orNone(up)) + } + case len(up) == 0: + return Found{}, ErrNone + case len(up) > 1: + return Found{}, fmt.Errorf("%w: %s. Name the one the predecessor's machines reach with --tunnel", + ErrSeveral, strings.Join(up, ", ")) + default: + iface = up[0] + } + + path := ConfigDir + "/" + iface + ".conf" + raw, err := ReadFile(path) + if err != nil { + return Found{}, fmt.Errorf("%s is up and its configuration cannot be read: %w", iface, err) + } + found, err := Parse(raw) + if err != nil { + return Found{}, fmt.Errorf("%s: %w", path, err) + } + found.Interface, found.Unit, found.Config = iface, "wg-quick@"+iface, path + return found, nil +} + +func orNone(names []string) string { + if len(names) == 0 { + return "none" + } + return strings.Join(names, ", ") +} + +// Parse reads a wg-quick configuration: the interface's key, port and address, and each peer's +// key and allowed address. Refused when it lacks what the mesh needs — a key, an address with a +// prefix — because a tunnel taken over without them is one the peers cannot reach. +func Parse(raw []byte) (Found, error) { + var f Found + section := "" + var peer *Peer + closePeer := func() error { + if peer == nil { + return nil + } + if peer.PublicKey == "" { + return errors.New("a [Peer] section has no PublicKey") + } + if peer.Address == "" { + return fmt.Errorf("the peer %s has no AllowedIPs, so the tunnel routes nothing to it", + short(peer.PublicKey)) + } + f.Peers = append(f.Peers, *peer) + peer = nil + return nil + } + for n, line := range strings.Split(string(raw), "\n") { + line = strings.TrimSpace(line) + if i := strings.IndexAny(line, "#;"); i >= 0 { + line = strings.TrimSpace(line[:i]) + } + if line == "" { + continue + } + if strings.HasPrefix(line, "[") { + if err := closePeer(); err != nil { + return Found{}, err + } + section = strings.ToLower(strings.Trim(line, "[]")) + if section == "peer" { + peer = &Peer{} + } + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + return Found{}, fmt.Errorf("line %d is not `key = value`", n+1) + } + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + switch section { + case "interface": + switch key { + case "privatekey": + f.privateKey = value + case "listenport": + port, err := strconv.Atoi(value) + if err != nil || port < 1 || port > 65535 { + return Found{}, fmt.Errorf("ListenPort %q is not a port", value) + } + f.Port = port + case "address": + // The first address is the interface's; a second family would be a second + // tunnel's worth of addressing, which this does not carry. + first := strings.TrimSpace(strings.Split(value, ",")[0]) + ip, network, err := net.ParseCIDR(first) + if err != nil { + return Found{}, fmt.Errorf("Address %q is not an address with a prefix length, "+ + "and the range the mesh takes over is read from the prefix", first) + } + f.Address = first + f.Range = network.String() + _ = ip + } + case "peer": + switch key { + case "publickey": + peer.PublicKey = value + case "allowedips": + peer.Address = strings.TrimSpace(strings.Split(value, ",")[0]) + case "endpoint": + peer.Endpoint = value + } + } + } + if err := closePeer(); err != nil { + return Found{}, err + } + if f.privateKey == "" { + return Found{}, errors.New("no PrivateKey in [Interface]; the mesh takes a tunnel over with its key or not at all") + } + if f.Address == "" { + return Found{}, errors.New("no Address in [Interface], so neither the hub's address nor the range can be read") + } + if f.Port == 0 { + return Found{}, errors.New("no ListenPort in [Interface]: a tunnel with no port is one nothing dials, so there is nothing to take over") + } + public, err := PublicKeyOf(f.privateKey) + if err != nil { + return Found{}, err + } + f.PublicKey = public + return f, nil +} + +// PublicKeyOf derives the public half of a WireGuard private key, both base64. +func PublicKeyOf(privateBase64 string) (string, error) { + raw, err := base64.StdEncoding.DecodeString(privateBase64) + if err != nil { + return "", fmt.Errorf("the private key is not base64: %w", err) + } + private, err := ecdh.X25519().NewPrivateKey(raw) + if err != nil { + return "", fmt.Errorf("the private key is not a Curve25519 key: %w", err) + } + return base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), nil +} + +func short(key string) string { + if len(key) > 8 { + return key[:8] + "…" + } + return key +} diff --git a/internal/tunnel/tunnel_test.go b/internal/tunnel/tunnel_test.go new file mode 100644 index 0000000..7272a6b --- /dev/null +++ b/internal/tunnel/tunnel_test.go @@ -0,0 +1,172 @@ +package tunnel + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" +) + +// novox/hq ADR 0105: the host reads the predecessor's tunnel — key, port, address and range, every +// peer — and the private key becomes the node's, never printed and never sent. + +// aKey is a real WireGuard keypair, made here so a key that stopped being a key is caught. +func aKey(t *testing.T) (private, public string) { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(k.Bytes()), + base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()) +} + +func aConfig(private string, peers ...string) string { + var b strings.Builder + fmt.Fprintf(&b, "# the predecessor's hub\n[Interface]\nPrivateKey = %s\nListenPort = 51900\n"+ + "Address = 192.0.2.1/24\n", private) + for i, key := range peers { + fmt.Fprintf(&b, "\n[Peer]\nPublicKey = %s\nAllowedIPs = 192.0.2.%d/32\n", key, i+2) + } + return b.String() +} + +func TestTheConfigurationIsReadWhole(t *testing.T) { + private, public := aKey(t) + _, peerA := aKey(t) + _, peerB := aKey(t) + found, err := Parse([]byte(aConfig(private, peerA, peerB) + "PersistentKeepalive = 25 ; a comment\n")) + if err != nil { + t.Fatal(err) + } + if found.Port != 51900 || found.Address != "192.0.2.1/24" || found.Range != "192.0.2.0/24" { + t.Errorf("port, address or range misread: %+v", found) + } + if found.PublicKey != public { + t.Errorf("the public key is not the one derived from the file's private key") + } + if found.PrivateKey() != private { + t.Error("the private key was not read") + } + if len(found.Peers) != 2 || found.Peers[0].PublicKey != peerA || found.Peers[0].Address != "192.0.2.2/32" || + found.Peers[1].PublicKey != peerB || found.Peers[1].Address != "192.0.2.3/32" { + t.Errorf("the peers were misread: %+v", found.Peers) + } +} + +func TestThePrivateKeyNeverPrintsAndNeverTravels(t *testing.T) { + private, _ := aKey(t) + found, err := Parse([]byte(aConfig(private))) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(found) + if err != nil { + t.Fatal(err) + } + for what, said := range map[string]string{ + "JSON": string(raw), + "String": found.String(), + "%v": fmt.Sprintf("%v", found), + "%+v": fmt.Sprintf("%+v", found), + "%#v via %v": fmt.Sprintf("%v", []Found{found}), + } { + if strings.Contains(said, private) { + t.Errorf("the private key appears in %s: %s", what, said) + } + } + if !strings.Contains(string(raw), found.PublicKey) { + t.Error("the public key does not travel, so the mesh could not know the tunnel's key") + } +} + +func TestATunnelWithoutWhatTheMeshNeedsIsRefused(t *testing.T) { + private, _ := aKey(t) + _, peer := aKey(t) + for name, conf := range map[string]string{ + "no key": "[Interface]\nListenPort = 51900\nAddress = 192.0.2.1/24\n", + "no address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\n", private), + "bare address": fmt.Sprintf("[Interface]\nPrivateKey = %s\nListenPort = 51900\nAddress = 192.0.2.1\n", private), + "no port": fmt.Sprintf("[Interface]\nPrivateKey = %s\nAddress = 192.0.2.1/24\n", private), + "peer no route": aConfig(private) + "\n[Peer]\nPublicKey = " + peer + "\n", + "peer no key": aConfig(private) + "\n[Peer]\nAllowedIPs = 192.0.2.9/32\n", + "not a key": "[Interface]\nPrivateKey = not-base64!\nListenPort = 1\nAddress = 192.0.2.1/24\n", + } { + if _, err := Parse([]byte(conf)); err == nil { + t.Errorf("%s was accepted", name) + } + } +} + +// aMachine answers `wg show interfaces` and reads configurations from a map. +type aMachine struct { + up string + files map[string]string + asked []string +} + +func (m *aMachine) run(_ context.Context, name string, args ...string) (string, error) { + m.asked = append(m.asked, name+" "+strings.Join(args, " ")) + if name == "wg" && len(args) == 2 && args[0] == "show" && args[1] == "interfaces" { + return m.up, nil + } + return "", errors.New("unexpected: " + name) +} + +func (m *aMachine) read(path string) ([]byte, error) { + if raw, ok := m.files[path]; ok { + return []byte(raw), nil + } + return nil, errors.New("no such file: " + path) +} + +func TestTheOneTunnelUpBesidesTheMeshsIsFound(t *testing.T) { + private, public := aKey(t) + m := &aMachine{up: "mesh0 wg0\n", files: map[string]string{ConfigDir + "/wg0.conf": aConfig(private)}} + ReadFile = m.read + t.Cleanup(func() { ReadFile = os.ReadFile }) + + found, err := Find(context.Background(), m.run, "") + if err != nil { + t.Fatal(err) + } + if found.Interface != "wg0" || found.Unit != "wg-quick@wg0" || found.Config != ConfigDir+"/wg0.conf" || + found.PublicKey != public { + t.Errorf("the wrong tunnel, or misnamed: %+v", found) + } + for _, asked := range m.asked { + if strings.HasPrefix(asked, "wg set") || strings.Contains(asked, "private-key") { + t.Errorf("finding a tunnel ran %q; reading is reading", asked) + } + } +} + +func TestNoneUpIsAnOrdinaryAnswerAndSeveralIsAQuestion(t *testing.T) { + private, _ := aKey(t) + m := &aMachine{up: "mesh0\n", files: map[string]string{ + ConfigDir + "/wg0.conf": aConfig(private), ConfigDir + "/wg1.conf": aConfig(private)}} + ReadFile = m.read + t.Cleanup(func() { ReadFile = os.ReadFile }) + + if _, err := Find(context.Background(), m.run, ""); !errors.Is(err, ErrNone) { + t.Errorf("a machine with only the mesh's interface up was not an ordinary none: %v", err) + } + m.up = "wg1 mesh0 wg0\n" + _, err := Find(context.Background(), m.run, "") + if !errors.Is(err, ErrSeveral) || !strings.Contains(err.Error(), "wg0, wg1") || strings.Contains(err.Error(), "mesh0") { + t.Errorf("two tunnels up were not refused naming both and only them: %v", err) + } + found, err := Find(context.Background(), m.run, "wg1") + if err != nil || found.Interface != "wg1" { + t.Errorf("naming one of two did not find it: %+v %v", found, err) + } + if _, err := Find(context.Background(), m.run, "wg9"); err == nil || !strings.Contains(err.Error(), "wg9") { + t.Errorf("naming a tunnel that is not up was not refused: %v", err) + } +}