diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index a8f420a..b38c6a1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -56,6 +56,8 @@ const usage = `mesh-host — the node host apply FILE make this machine match a declaration from a file reconcile make this machine match the declaration this host carries bundle show what this host carries + overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this + node's overlay key and the mesh is told, signed; --tunnel when several are up owned what this host has applied and still owns version @@ -147,6 +149,13 @@ func parseArgs(args []string) (string, options, error) { opts.file = positionals[0] return command, opts, nil } + if command == "overlay" { + if len(positionals) != 1 { + return "", opts, errors.New("overlay take [--tunnel ]") + } + opts.file = positionals[0] + return command, opts, nil + } // Anything left over was neither the command nor a flag. Refused rather than ignored: a // mistyped argument that changes nothing and reports success is worse than an error. if len(positionals) > 0 { @@ -234,6 +243,8 @@ func run(ctx context.Context, command string, opts options) error { case "enrol", "enroll": return enrol(ctx, opts) + case "overlay": + return overlayCommand(ctx, opts) case "run": return runLink(ctx, opts) @@ -608,6 +619,80 @@ func enrol(ctx context.Context, opts options) error { return nil } +// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over +// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a +// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them +// every credential the mesh sealed to it. +func overlayCommand(ctx context.Context, opts options) error { + if opts.file != "take" { + return errors.New("overlay take [--tunnel ] — the one thing `overlay` does here") + } + identityPath := identity.Path(opts.state) + mine, err := identity.Load(identityPath) + if err != nil { + return err + } + found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) + if err != nil { + return fmt.Errorf("%w. Nothing was changed", err) + } + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + return err + } + fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public) + fmt.Printf(" identity, sealing and serving keys are untouched\n") + + // Told first, then written: a mesh told and a machine not yet written is put right by running + // this again (the mesh refuses the stale second rekey and changes nothing; the files are + // rewritten the same). A machine written and a mesh not told would raise the mesh's interface + // on a key the mesh does not know at the next restart. + if err := link.Publish(ctx, link.Membership{ + Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, + Password: mine.Membership.Password, Signer: mine.Membership.Signer, + }, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil { + return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err) + } + fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node) + + if err := os.WriteFile(identity.OverlayKeyPath(opts.state), + []byte(taken.Overlay.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err) + } + if err := identity.Save(identityPath, taken); err != nil { + return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err) + } + fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n", + identity.OverlayKeyPath(opts.state)) + fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint :%d …`, `plan %s --json`, then push\n", + mine.Node, found.Port, mine.Node) + return nil +} + +// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey +// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same +// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names +// is the one before the take, so the mesh can tell a repeat from a replay. +func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) { + overlay, err := identity.OverlayKeyFrom(found.PrivateKey()) + if err != nil { + return identity.Identity{}, link.Rekey{}, err + } + previous := mine.Overlay.Public + if previous == overlay.Public && mine.OverlayBefore != "" { + previous = mine.OverlayBefore + } + taken := mine + taken.Overlay = overlay + if previous != overlay.Public { + taken.OverlayBefore = previous + } + presented := carried(found) + rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented} + rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented)) + return taken, rekey, nil +} + // carried is a found tunnel as it is presented to the mesh: everything but its private key. func carried(t tunnel.Found) *link.Tunnel { out := &link.Tunnel{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, @@ -934,7 +1019,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // And the tunnel the private network took over, as this apply found it (novox/hq ADR 0105). if t := outcome.Tunnel; t != nil { report.Tunnel = &link.CarriedTunnel{Interface: t.Interface, Port: t.Port, Range: t.Range, - Peers: t.Peers, Taken: t.Taken, Kept: t.Kept} + Peers: t.Peers, State: t.State, Note: t.Note, Kept: t.Kept} } reached, err := reachable.Collect(ctx, apply.ExecRunner) if err != nil { diff --git a/cmd/mesh-host/rekey_test.go b/cmd/mesh-host/rekey_test.go new file mode 100644 index 0000000..7b19edd --- /dev/null +++ b/cmd/mesh-host/rekey_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "crypto/ed25519" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/tunnel" +) + +// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and +// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a +// proof signed by the identity key, over the previous key, the new one and the tunnel. + +func anEnrolledNode(t *testing.T) identity.Identity { + t.Helper() + mine, err := identity.Generate("anchor") + if err != nil { + t.Fatal(err) + } + mine.Overlay, err = identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa", + Signer: make([]byte, ed25519.PublicKeySize), Password: "p"} + return mine +} + +func aFoundTunnel(t *testing.T) tunnel.Found { + t.Helper() + private, err := identity.GenerateOverlayKey() + if err != nil { + t.Fatal(err) + } + found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" + + "Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n")) + if err != nil { + t.Fatal(err) + } + found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf" + return found +} + +func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) { + mine := anEnrolledNode(t) + found := aFoundTunnel(t) + before := mine.Overlay.Public + + taken, rekey, err := rekeyOnto(mine, found) + if err != nil { + t.Fatal(err) + } + if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() { + t.Fatal("the overlay key is not the tunnel's") + } + if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) || + taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password || + taken.Membership.Broker != mine.Membership.Broker { + t.Fatal("something other than the overlay key moved") + } + if taken.OverlayBefore != before { + t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore) + } + if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil || + rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 { + t.Fatalf("the rekey does not say what moved: %+v", rekey) + } + if !ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the rekey is not signed by this node's identity key over what it says") + } + if ed25519.Verify(ed25519.PublicKey(mine.Public), + link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { + t.Fatal("the proof is not bound to the node") + } + for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} { + if strings.Contains(said, found.PrivateKey()) { + t.Fatal("the private key travels") + } + } + + // Run again after the take — the mesh not yet told, or told and refused — the previous key it + // names is still the one before the take, so the mesh can tell a repeat from a replay. + again, second, err := rekeyOnto(taken, found) + if err != nil { + t.Fatal(err) + } + if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey { + t.Fatalf("a take run again does not name the key before the first: %+v", second) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index c2fd918..6c4309b 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -339,22 +339,32 @@ func ApplyKeeping( // it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never // flushed. A failure here fails the service too — the mesh's interface is not started on a // port the found one still holds. + stoppedFound := false if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil { var outcome Outcome var facts TakenTunnel var err error if d.Adoption == nil { err = errNotAdopted + facts = TakenTunnel{Interface: svc.TakesOver.Interface, State: NotTaken} } else { - outcome, facts, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) + outcome, facts, stoppedFound, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC()) } + // Always an account, failure included: the last account standing must never be an + // older "taken" over a machine whose takeover has since gone wrong. + report.Tunnel = &facts if err != nil { + report.Tunnel.Note = err.Error() + if stoppedFound { + // The found unit is down and the mesh's not up: the one state where the + // peers reach nothing. Started again, and said. + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report}) log(fmt.Sprintf(" failed %s (%s): %v", svc.Identity(), svc.Unit, err)) continue } report.Outcomes = append(report.Outcomes, outcome) - report.Tunnel = &facts log(fmt.Sprintf(" held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail)) } @@ -377,6 +387,17 @@ func ApplyKeeping( failed := &Error{Resource: resource.Identity(), Err: err, Done: report} failures = append(failures, failed) log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err)) + if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { + // The mesh's interface did not come up after the found one was stopped: no + // tunnel at all. The found unit is started again — the machine goes back to + // what it had — and the account says so (novox/hq ADR 0105). + report.Tunnel.Note = "the mesh's interface did not come up: " + err.Error() + if stoppedFound { + restoreFound(ctx, sys, svc.TakesOver.Unit, run, report.Tunnel) + } else { + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) + } + } // **A failed action stops what follows. Nothing else does.** // @@ -426,8 +447,8 @@ func ApplyKeeping( } if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil { // The found interface is down and the mesh's is up in its place: the tunnel changed - // hands (novox/hq ADR 0105). - report.Tunnel.Taken = true + // hands (novox/hq ADR 0105). Read from the machine, not assumed. + report.Tunnel.State = tunnelState(ctx, sys, svc.TakesOver.Unit, svc.Unit, run) } report.Outcomes = append(report.Outcomes, outcome) if outcome.Action != "unchanged" { diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 3e95e36..be2277c 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -31,6 +31,8 @@ type machine struct { type fakeUnit struct { active, enabled string + // wontStart is a unit that accepts `start` and stays inactive — one that starts and dies. + wontStart bool // fragment is where systemd loads the unit from; empty means /etc/systemd/system, where an // administrator installs one. fragment string @@ -65,7 +67,9 @@ func (m *machine) systemctl(args []string) (string, error) { } return u.enabled + "\n", nil case "start": - u.active = "active" + if !u.wontStart { + u.active = "active" + } case "stop": u.active = "inactive" case "enable": diff --git a/internal/apply/takeover.go b/internal/apply/takeover.go index bff4935..359412c 100644 --- a/internal/apply/takeover.go +++ b/internal/apply/takeover.go @@ -34,26 +34,53 @@ type TakenTunnel struct { Port int Range string Peers int - // Taken is whether the found interface is down and disabled and the mesh's up in its place. - Taken bool + // State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one + // down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's + // not up: the peers reach nothing). Note is what this apply did about it. + State string + Note string Kept string } +// The states, as the link says them. +const ( + NotTaken = "not-taken" + Taken = "taken" + TunnelDown = "down" +) + +// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up +// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a +// person takes a moment. Variables so a test need not wait. +var ( + takeoverRecheck = 2 * time.Second + takeoverRechecks = 3 +) + // takeOverID is the held record's id for the found configuration: the service's own with a suffix, // so it is declared for as long as the service is and never mistaken for the service itself. func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" } // takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that // replaces it. Returned is the hold's outcome, and what was found for the report. +// +// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's +// option 2 is exactly this going wrong): the declared configuration must listen on the found port +// at the found address, and the key file it points at must hold the found key. Only then is the +// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then +// fails to start can have the found unit started again. func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration, - known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) { + known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) { t := svc.TakesOver id := takeOverID(svc) module, _ := d.Adoption.UntakenModuleOf(svc.ID) if module == "" { module = "the private network" } - facts := TakenTunnel{Interface: t.Interface} + facts = TakenTunnel{Interface: t.Interface, State: NotTaken} + + // 0. What the found configuration says, before anything: the checks below are against it. + found, ferr := readFoundTunnel(t.Config) // 1. The configuration, kept like any held file. A synthetic file resource stands for it, so // the same code keeps its original, digests it and notices it changing. @@ -62,65 +89,97 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, out, held, err := hold(ctx, sys, file, module, was, already, "the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now) if err != nil { - return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err) + return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err) } known.RecordHeld(held) facts.Kept = held.Kept - // What the file says, for the report: read from the machine, or from the kept original when - // the machine's copy is gone. The private key stays in the file; nothing here keeps it. + // What the file says, for the report: from the machine, or from the kept original when the + // machine's copy is gone. The private key stays in the file; nothing here keeps it. unread := "" - raw, err := os.ReadFile(t.Config) - if err != nil && held.Kept != "" { - raw, err = os.ReadFile(held.Kept) + if ferr != nil && held.Kept != "" { + found, ferr = readFoundTunnel(held.Kept) } - if err == nil { - if found, perr := tunnel.Parse(raw); perr == nil { - facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) - } else { - unread = perr.Error() - } + if ferr == nil { + facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers) } else { - unread = err.Error() + unread = ferr.Error() } - // 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not - // there is not an error — the interface may have been raised another way, which the check - // below catches — and neither is one already down. + // 2. Where things stand: the found unit, and the mesh's. + foundState, unitErr := sys.ServiceState(ctx, run, t.Unit) + meshState, _ := sys.ServiceState(ctx, run, svc.Unit) + if foundState == "running" && meshState == "running" { + // Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's + // holds — and on a spoke two interfaces with one key flap between them. Not stopped again + // by the mesh: what is found on an adopted node is reported, and the first takeover was + // the one act (the PR note says why). Said, so a person sees it. + facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " + + "`systemctl stop " + t.Unit + "` on the machine" + } + + // 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared + // configuration must listen on the found port at the found address, and the key file it + // points at must hold the found key, or the peers would be dropped the moment it came up. + if foundState == "running" && meshState != "running" { + if ferr != nil { + return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+ + "tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running", + t.Config, ferr, t.Unit) + } + if err := replaces(d, svc, found); err != nil { + return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit) + } + } + + // 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at + // boot. A unit that is not there is not an error — the interface may have been raised + // another way, which the check below catches — and neither is one already down. var did []string - state, err := sys.ServiceState(ctx, run, t.Unit) switch { - case err != nil: + case unitErr != nil: did = append(did, t.Unit+" is not a unit here") - case state == "running": + case foundState == "running" && meshState != "running": if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil { - return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err) + return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err) } after, err := sys.ServiceState(ctx, run, t.Unit) if err != nil { - return out, facts, err + return out, facts, true, err } if after != "stopped" { - return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) + return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after) } + stopped = true did = append(did, "stopped "+t.Unit) } - if err == nil { + if unitErr == nil { if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" { if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil { - return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) + return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err) } did = append(did, "disabled it at boot") } } - // 3. The interface is gone. If it is still up, something other than its unit raised it, and - // starting the mesh's on the same port and address would fail or, worse, half work. - if up, err := run(ctx, "wg", "show", "interfaces"); err == nil { - for _, iface := range strings.Fields(up) { - if iface == t.Interface { - return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+ - "than that unit raises it, and the mesh's interface cannot take its port and address "+ - "while it does. Nothing was flushed", t.Interface, t.Unit) + // 5. The interface is gone. If it is still up, something other than its unit raised it — + // the predecessor brings its up by hand — and the mesh's interface cannot take its port + // and address while it is. Looked at again for a moment, since a person taking it down + // takes a moment; then refused, naming what to do. + if meshState != "running" { + for try := 0; ; try++ { + if !interfaceUp(ctx, run, t.Interface) { + break + } + if try >= takeoverRechecks { + return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+ + "raised by hand, not by its unit, and the mesh's interface cannot take its port and "+ + "address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+ + "over. Nothing was flushed", t.Interface, t.Unit, t.Interface) + } + select { + case <-ctx.Done(): + return out, facts, stopped, ctx.Err() + case <-time.After(takeoverRecheck): } } } @@ -140,7 +199,140 @@ func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, // peers was carried, which reads as a tunnel that was not one. out.Detail += "; what it says could not be read as a tunnel's: " + unread } - return out, facts, nil + return out, facts, stopped, nil +} + +// readFoundTunnel is the found configuration as a tunnel. +func readFoundTunnel(path string) (tunnel.Found, error) { + raw, err := os.ReadFile(path) + if err != nil { + return tunnel.Found{}, err + } + return tunnel.Parse(raw) +} + +// interfaceUp is whether a WireGuard interface is up on the machine. +func interfaceUp(ctx context.Context, run Runner, iface string) bool { + up, err := run(ctx, "wg", "show", "interfaces") + if err != nil { + return false + } + for _, name := range strings.Fields(up) { + if name == iface { + return true + } + } + return false +} + +// replaces holds the mesh's declared interface configuration against the found tunnel it is to +// replace: same port, same address, and a key file holding the found key. The configuration is +// the file the service restarts on; its `PostUp = wg set %i private-key ` names the key. +func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error { + var conf *declaration.File + for _, r := range d.Resources { + f, ok := r.(*declaration.File) + if !ok { + continue + } + for _, id := range svc.RestartOn { + if f.ID == id { + conf = f + } + } + } + if conf == nil { + return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+ + "raise cannot be checked against the found one", svc.Unit, found.Interface) + } + port, address, keyPath := "", "", "" + for _, line := range strings.Split(conf.Content, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + switch key { + case "listenport": + port = value + case "address": + address = strings.TrimSpace(strings.Split(value, ",")[0]) + case "postup": + if _, after, ok := strings.Cut(value, "private-key "); ok { + keyPath = strings.Fields(after)[0] + } + } + } + var wrong []string + if port != fmt.Sprint(found.Port) { + wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port)) + } + if host(address) != host(found.Address) { + wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address)) + } + switch raw, err := os.ReadFile(keyPath); { + case keyPath == "": + wrong = append(wrong, "it names no key file") + case err != nil: + wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err)) + default: + public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw))) + if perr != nil || public != found.PublicKey { + wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+ + "--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface)) + } + } + if len(wrong) > 0 { + return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+ + "dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again", + found.Interface, strings.Join(wrong, "; ")) + } + return nil +} + +// host is an address without its prefix length. +func host(address string) string { + if i := strings.Index(address, "/"); i >= 0 { + return address[:i] + } + return address +} + +// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up +// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is +// down — the peers reach nothing. +func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string { + foundState, _ := sys.ServiceState(ctx, run, foundUnit) + meshState, _ := sys.ServiceState(ctx, run, meshUnit) + foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@")) + switch { + case meshState == "running" && !foundUp: + return Taken + case meshState == "running": + // Both up: not a takeover that holds, and said as not taken so nobody reads it as one. + return NotTaken + case foundUp: + return NotTaken + default: + return TunnelDown + } +} + +// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the +// machine has the tunnel it had rather than none, and says so in the account. +func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) { + if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil { + facts.State = TunnelDown + facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit + return + } + if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" { + facts.State = TunnelDown + facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit + return + } + facts.State = NotTaken + facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had" } // takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since diff --git a/internal/apply/takeover_test.go b/internal/apply/takeover_test.go index 2ca9abf..18fde16 100644 --- a/internal/apply/takeover_test.go +++ b/internal/apply/takeover_test.go @@ -14,53 +14,63 @@ import ( ) // novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the -// found interface without flushing it, and keeps its configuration. +// found interface without flushing it, and keeps its configuration — and stops nothing until the +// mesh's interface is known to be able to replace it. -// foundConf is the predecessor's configuration, with a real key made once per run: the key is -// what the takeover must never print or copy, so it had better be one. -var foundConf = func() string { +// foundKey is the predecessor's private key, a real one made once per run: the key is what the +// takeover must never print or copy, so it had better be one. +var foundKey = func() string { k, err := ecdh.X25519().GenerateKey(rand.Reader) if err != nil { panic(err) } - return "[Interface]\nPrivateKey = " + base64.StdEncoding.EncodeToString(k.Bytes()) + "\n" + - "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + - "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" + return base64.StdEncoding.EncodeToString(k.Bytes()) }() +var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" + + "ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" + + "\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n" + // aTakeover is the private network's declaration for an adopted hub whose interface takes over -// the found tunnel: the mesh's configuration — with the found key set from the node's own key file -// and the found peers in its list — and the interface's service naming what it replaces. -func aTakeover(t *testing.T, config, mesh string) *declaration.Declaration { +// the found tunnel: the mesh's configuration — on the found port and address, its key set from the +// node's own key file, the found peers in its list — and the interface's service naming what it +// replaces. Port and address are parameters so a test can declare a wrong one. +func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration { t.Helper() return adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`, `{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600", - "content":"[Interface]\nAddress = 192.0.2.1/32\nListenPort = 51900\nPostUp = wg set %i private-key /var/lib/mesh-host/overlay.key\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, + "content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"}, {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled", "restart-on":["mesh-wireguard.overlay-config"], "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`) } -// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet. -func aHubInUse(t *testing.T) (dir, config, mesh string, m *machine) { +// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found +// configuration on disk, and the node's key file holding the found key, as enrolment left it. +func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) { t.Helper() dir = t.TempDir() config = filepath.Join(dir, "wg0.conf") mesh = filepath.Join(dir, "mesh0.conf") + keyFile = filepath.Join(dir, "overlay.key") if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil { t.Fatal(err) } + if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil { + t.Fatal(err) + } m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{ "wg-quick@wg0": {active: "active", enabled: "enabled"}, "wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"}, }} - return dir, config, mesh, m + takeoverRecheck = 0 + return dir, config, mesh, keyFile, m } func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) - report, state := applyAdopted(t, aTakeover(t, config, mesh), store.State{}, m, dir) + dir, config, mesh, keyFile, m := aHubInUse(t) + report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir) // The found interface: its unit stopped and disabled, and nothing else done to it. if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" { @@ -93,16 +103,18 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got) } // And the report says so, with what was found — port, range, peers — and never the key. - if report.Tunnel == nil || !report.Tunnel.Taken || report.Tunnel.Port != 51900 || + if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 || report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept { t.Fatalf("the report does not say what was carried: %+v", report.Tunnel) } - private := strings.TrimSpace(strings.SplitN(strings.SplitN(foundConf, "PrivateKey = ", 2)[1], "\n", 2)[0]) for _, o := range report.Outcomes { - if strings.Contains(o.Detail, private) { + if strings.Contains(o.Detail, foundKey) { t.Errorf("the found key was printed in an outcome: %+v", o) } } + if strings.Contains(report.Tunnel.Note, foundKey) { + t.Error("the found key was printed in the account") + } if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" || !strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") { t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o) @@ -112,9 +124,67 @@ func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T } } -func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) - d := aTakeover(t, config, mesh) +func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + otherKey := filepath.Join(dir, "other.key") + k, _ := ecdh.X25519().GenerateKey(rand.Reader) + if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cases := map[string]*declaration.Declaration{ + "another port": aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"), + "another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"), + "another key": aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"), + "no key file": aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"), + } + for name, d := range cases { + m.asked = nil + report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{}, + store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") { + t.Fatalf("%s: the takeover was not refused: %v", name, err) + } + if name == "another key" && !strings.Contains(err.Error(), "overlay take") { + t.Errorf("%s: the refusal does not name the remedy: %v", name, err) + } + if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") { + t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name) + } + if m.units["wg-quick@mesh0"].active == "active" { + t.Fatalf("%s: the mesh's interface was started on top of the found one", name) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") { + t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel) + } + if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { + t.Errorf("%s: the found configuration was not kept before the refusal", name) + } + } +} + +func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + m.units["wg-quick@mesh0"].wontStart = true + report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil { + t.Fatal("a mesh interface that did not come up was reported as applied") + } + if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") { + t.Fatalf("the found unit was not stopped and then started again: %v", m.asked) + } + if m.units["wg-quick@wg0"].active != "active" { + t.Fatal("the machine was left with no tunnel at all") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || + !strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") { + t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel) + } +} + +func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1") _, state := applyAdopted(t, d, store.State{}, m, dir) m.asked = nil @@ -128,28 +198,49 @@ func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) { if m.did("systemctl stop wg-quick@wg0") { t.Error("a found unit already down was stopped again") } + if report.Tunnel == nil || report.Tunnel.State != Taken { + t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel) + } - // Somebody starts the found unit again: it would take the port back, so it is stopped again - // — the one thing on an adopted node the mesh undoes, because the tunnel is the mesh's now. + // Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh — + // on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said. m.units["wg-quick@wg0"].active = "active" m.asked = nil - _, _ = applyAdopted(t, d, again, m, dir) - if m.units["wg-quick@wg0"].active != "inactive" || !m.did("systemctl stop wg-quick@wg0") { - t.Error("a found unit started again was left holding the mesh's port") + report, _ = applyAdopted(t, d, again, m, dir) + if m.did("systemctl stop wg-quick@wg0") { + t.Error("a found unit started again by hand was stopped by the mesh") + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") { + t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel) } } -func TestAFoundInterfaceStillUpAfterItsUnitStoppedRefusesTheTakeover(t *testing.T) { - dir, config, mesh, m := aHubInUse(t) +func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) { + dir, config, mesh, keyFile, m := aHubInUse(t) + // The unit is not running, yet the interface is up: the predecessor raised it by hand. + m.units["wg-quick@wg0"].active = "inactive" m.wgUp = "wg0 mesh0\n" - _, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh), store.State{}, - store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) - if err == nil || !strings.Contains(err.Error(), "still up") || !strings.Contains(err.Error(), "Nothing was flushed") { - t.Fatalf("an interface something else raises was taken over anyway: %v", err) + report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), + store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir)) + if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") { + t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err) } if m.units["wg-quick@mesh0"].active == "active" { t.Error("the mesh's interface was started on a port the found one still holds") } + // Looked at more than once before giving up: a person taking it down takes a moment. + shows := 0 + for _, a := range m.asked { + if a == "wg show interfaces" { + shows++ + } + } + if shows < takeoverRechecks+1 { + t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows) + } + if report.Tunnel == nil || report.Tunnel.State != NotTaken { + t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel) + } if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held { t.Error("the found configuration was not kept before the refusal") } diff --git a/internal/identity/identity.go b/internal/identity/identity.go index 9ff2ee8..c8d7430 100644 --- a/internal/identity/identity.go +++ b/internal/identity/identity.go @@ -49,8 +49,13 @@ type Identity struct { Membership Membership `json:"membership"` // Overlay is this node's key on the private network. Generated here, like the identity above, - // and for the same reason: the mesh computes a graph it cannot impersonate. + // and for the same reason: the mesh computes a graph it cannot impersonate — or, on an adopted + // node that took a found tunnel over, that tunnel's key (novox/hq ADR 0105). Overlay OverlayKey `json:"overlay"` + // OverlayBefore is the public half of the overlay key this node held before it took a found + // tunnel's, so a take run again names the key the mesh still records. Empty on a node that + // never took one. + OverlayBefore string `json:"overlay_before,omitempty"` } // Membership is how this node reaches the mesh it belongs to, and who it believes. diff --git a/internal/link/messages.go b/internal/link/messages.go index fe27ccb..76765a0 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,6 +1,10 @@ package link -import "time" +import ( + "strconv" + "strings" + "time" +) // The wire formats shared with the control plane, which defines them separately because this // binary requires nothing present and does not import it. A test on each side asserts the field @@ -105,18 +109,64 @@ type Report struct { // 0105): which interface, its port, range and peer count, whether the found interface is down // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` + + // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq + // ADR 0105). Not an account of the machine: a report carrying one says nothing else. + Rekey *Rekey `json:"rekey,omitempty"` } -// CarriedTunnel is this node's account of the tunnel it took over. +// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried +// states below; Note is what the host did about it, when it did something. type CarriedTunnel struct { Interface string `json:"interface"` Port int `json:"port"` Range string `json:"range"` Peers int `json:"peers"` - Taken bool `json:"taken"` + State string `json:"state"` + Note string `json:"note,omitempty"` Kept string `json:"kept,omitempty"` } +// The states a carried tunnel can be in: the found interface still up and the mesh's not; the +// found one down and the mesh's up with its key; or the found one down and the mesh's not up — the +// one state where the peers reach nothing, said as its own word so nothing reads it as either of +// the others. +const ( + CarriedNotTaken = "not-taken" + CarriedTaken = "taken" + CarriedDown = "down" +) + +// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling +// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel +// over, since re-enrolling would rotate every key it holds. Signed with the identity key over +// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. +type Rekey struct { + // Previous is the overlay key this node held until now, as the mesh records it; the mesh + // refuses a rekey naming another, which is how a replayed one is refused. + Previous string `json:"previous"` + OverlayKey string `json:"overlay_key"` + Tunnel *Tunnel `json:"tunnel"` + Proof []byte `json:"proof"` +} + +// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes +// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. +// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). +func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { + var t Tunnel + if tunnel != nil { + t = *tunnel + } + peers := make([]string, 0, len(t.Peers)) + for _, p := range t.Peers { + peers = append(peers, p.PublicKey+"@"+p.Address) + } + return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) +} + // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` diff --git a/internal/link/run.go b/internal/link/run.go index 1379369..2675065 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -377,6 +377,39 @@ func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) R } // publishReport tells the mesh what this node did, and says whether the broker took it. +// Publish sends one report on this node's own connection and returns: the one-shot path for a +// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same +// account, the same pinned certificate and the same exchange as the running host's reports. +func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error { + config, err := PinnedConfig(m.Fingerprint) + if err != nil { + return err + } + dsn := fmt.Sprintf("amqps://%s:%s@%s/", + url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker) + conn, err := amqp.DialConfig(dsn, amqp.Config{ + TLSClientConfig: config, + Dial: amqp.DefaultDial(timeout), + }) + if err != nil { + if errors.Is(err, ErrWrongCertificate) { + return err + } + return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err) + } + defer conn.Close() + channel, err := conn.Channel() + if err != nil { + return err + } + defer channel.Close() + var said string + if !publishReport(ctx, channel, m, report, func(s string) { said = s }, timeout) { + return errors.New(said) + } + return nil +} + func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report, say Announce, timeout time.Duration) bool { report.Node = m.Node