Files
mesh-host/internal/apply/takeover.go
T
jochen b462f461c6 A taken tunnel's found configuration is retired once the take is proven (hq ADR 0119)
Kept on disk it was the take's fallback; once the mesh's interface is up in its place and a peer
has handshaken with it, it is an unmaintained way back onto the network, held for ever. It is now
removed from where its unit reads it, its kept original verified first and left as it is, and the
hold ends. Until proven — no handshake, or wg not answering — it is kept and the report says why.
The retirement is recorded apart from holds, so later applies, an undeclare, and a reassignment
find it retired rather than missing, and nothing writes it back.
2026-09-27 00:47:57 +02:00

506 lines
22 KiB
Go

package apply
import (
"context"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
"github.com/novox/mesh-host/internal/tunnel"
)
// The private network takes over the tunnel it found (novox/hq ADR 0105).
//
// The controller says so on the interface's service: `takes-over` names the found interface, the
// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps
// that file like any held file — the original recorded before anything else happens to it — and
// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the
// file is never written, and the found interface goes down the way its own unit takes it down.
// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found
// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands.
//
// Every apply, not once: a found unit somebody starts again would take the port back from the
// mesh's interface, so it is stopped again and said so. That is the one place an adopted node
// undoes something done by hand, and it is because the tunnel is the mesh's now.
//
// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119).
// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up,
// the found unit is started again and the peers never notice. That caution is spent once the
// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has
// handshaken with the mesh's interface. From then on a configuration nothing maintains, one
// command away from raising a second way onto the network, is not a rollback path but a door
// nobody watches. So it is removed from where its unit reads it; its original, kept before
// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven
// keeps it, and says so — a broken take is visible, not silently retired.
// TakenTunnel is what an apply says about a tunnel it took over, for the node's report.
type TakenTunnel struct {
Interface string
Port int
Range string
Peers int
// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
// not up: the peers reach nothing). Note is what this apply did about it — and, for a taken
// tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119).
// Kept is where the found configuration's original is, retired or not.
State string
Note string
Kept string
}
// The states, as the link says them.
const (
NotTaken = "not-taken"
Taken = "taken"
TunnelDown = "down"
)
// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up
// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a
// person takes a moment. Variables so a test need not wait.
var (
takeoverRecheck = 2 * time.Second
takeoverRechecks = 3
)
// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
// so it is declared for as long as the service is and never mistaken for the service itself.
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
// replaces it. Returned is the hold's outcome, and what was found for the report.
//
// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's
// option 2 is exactly this going wrong): the declared configuration must listen on the found port
// at the found address, and the key file it points at must hold the found key. Only then is the
// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then
// fails to start can have the found unit started again.
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) {
t := svc.TakesOver
id := takeOverID(svc)
module, _ := d.Adoption.UntakenModuleOf(svc.ID)
if module == "" {
module = "the private network"
}
facts = TakenTunnel{Interface: t.Interface, State: NotTaken}
// 0. What the found configuration says, before anything: the checks below are against it.
found, ferr := readFoundTunnel(t.Config)
// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
// the same code keeps its original, digests it and notices it changing.
//
// **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed
// it, so there is nothing to hold and nothing missing — only where its original is, which
// the retirement recorded. A hold still standing is let go: that is what retiring it meant.
// One put back at its path by a person is on the machine again, with no hold, and is found
// and kept afresh — the same content kept once, as any original is — and retired again by
// the first apply that finds the take still proven.
file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
var held store.Held
retired, wasRetired := known.RetiredAt(t.Config)
if wasRetired && !present(t.Config) {
known.Release(id)
held = store.Held{Kept: retired.Kept}
out = begin(file)
out.Action = "unchanged"
facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " +
"its original is kept at " + retired.Kept + " and the mesh never brings it back"
} else {
if wasRetired {
known.Unretire(t.Config)
}
was, already := known.HeldAt(id)
out, held, err = hold(ctx, sys, file, module, was, already,
"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
if err != nil {
return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
}
known.RecordHeld(held)
}
facts.Kept = held.Kept
// What the file says, for the report: from the machine, or from the kept original when the
// machine's copy is gone. The private key stays in the file; nothing here keeps it.
unread := ""
if ferr != nil && held.Kept != "" {
found, ferr = readFoundTunnel(held.Kept)
}
if ferr == nil {
facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
} else {
unread = ferr.Error()
}
// 2. Where things stand: the found unit, and the mesh's.
foundState, unitErr := sys.ServiceState(ctx, run, t.Unit)
meshState, _ := sys.ServiceState(ctx, run, svc.Unit)
if foundState == "running" && meshState == "running" {
// Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's
// holds — and on a spoke two interfaces with one key flap between them. Not stopped again
// by the mesh: what is found on an adopted node is reported, and the first takeover was
// the one act (the PR note says why). Said, so a person sees it.
facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " +
"`systemctl stop " + t.Unit + "` on the machine"
}
// 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared
// configuration must listen on the found port at the found address, and the key file it
// points at must hold the found key, or the peers would be dropped the moment it came up.
if foundState == "running" && meshState != "running" {
if ferr != nil {
return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+
"tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running",
t.Config, ferr, t.Unit)
}
if err := replaces(d, svc, found); err != nil {
return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit)
}
}
// 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at
// boot. A unit that is not there is not an error — the interface may have been raised
// another way, which the check below catches — and neither is one already down.
var did []string
switch {
case unitErr != nil:
did = append(did, t.Unit+" is not a unit here")
case foundState == "running" && meshState != "running":
if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
}
after, err := sys.ServiceState(ctx, run, t.Unit)
if err != nil {
return out, facts, true, err
}
if after != "stopped" {
return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
}
stopped = true
did = append(did, "stopped "+t.Unit)
}
if unitErr == nil {
if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
}
did = append(did, "disabled it at boot")
}
}
// 5. The interface is gone. If it is still up, something other than its unit raised it —
// the predecessor brings its up by hand — and the mesh's interface cannot take its port
// and address while it is. Looked at again for a moment, since a person taking it down
// takes a moment; then refused, naming what to do.
if meshState != "running" {
for try := 0; ; try++ {
if !interfaceUp(ctx, run, t.Interface) {
break
}
if try >= takeoverRechecks {
return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+
"raised by hand, not by its unit, and the mesh's interface cannot take its port and "+
"address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+
"over. Nothing was flushed", t.Interface, t.Unit, t.Interface)
}
select {
case <-ctx.Done():
return out, facts, stopped, ctx.Err()
case <-time.After(takeoverRecheck):
}
}
}
out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
if wasRetired && out.Action == "unchanged" {
out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven"
}
if held.Kept != "" {
out.Detail += " (original at " + held.Kept + ")"
}
if len(did) > 0 {
out.Detail += "; " + strings.Join(did, ", ") + " — never flushed"
}
if held.Changed != "" {
out.Detail += "; " + held.Changed + " by something other than the mesh since it was found"
}
if unread != "" {
// Said, not swallowed: the report would otherwise say a tunnel with no port and no
// peers was carried, which reads as a tunnel that was not one.
out.Detail += "; what it says could not be read as a tunnel's: " + unread
}
return out, facts, stopped, nil
}
// readFoundTunnel is the found configuration as a tunnel.
func readFoundTunnel(path string) (tunnel.Found, error) {
raw, err := os.ReadFile(path)
if err != nil {
return tunnel.Found{}, err
}
return tunnel.Parse(raw)
}
// interfaceUp is whether a WireGuard interface is up on the machine.
func interfaceUp(ctx context.Context, run Runner, iface string) bool {
up, err := run(ctx, "wg", "show", "interfaces")
if err != nil {
return false
}
for _, name := range strings.Fields(up) {
if name == iface {
return true
}
}
return false
}
// replaces holds the mesh's declared interface configuration against the found tunnel it is to
// replace: same port, same address, and a key file holding the found key. The configuration is
// the file the service restarts on; its `PostUp = wg set %i private-key <path>` names the key.
func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error {
var conf *declaration.File
for _, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok {
continue
}
for _, id := range svc.RestartOn {
if f.ID == id {
conf = f
}
}
}
if conf == nil {
return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+
"raise cannot be checked against the found one", svc.Unit, found.Interface)
}
port, address, keyPath := "", "", ""
for _, line := range strings.Split(conf.Content, "\n") {
key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
if !ok {
continue
}
key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
switch key {
case "listenport":
port = value
case "address":
address = strings.TrimSpace(strings.Split(value, ",")[0])
case "postup":
if _, after, ok := strings.Cut(value, "private-key "); ok {
keyPath = strings.Fields(after)[0]
}
}
}
var wrong []string
if port != fmt.Sprint(found.Port) {
wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port))
}
if host(address) != host(found.Address) {
wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address))
}
switch raw, err := os.ReadFile(keyPath); {
case keyPath == "":
wrong = append(wrong, "it names no key file")
case err != nil:
wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err))
default:
public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw)))
if perr != nil || public != found.PublicKey {
wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+
"--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface))
}
}
if len(wrong) > 0 {
return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+
"dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again",
found.Interface, strings.Join(wrong, "; "))
}
return nil
}
// host is an address without its prefix length.
func host(address string) string {
if i := strings.Index(address, "/"); i >= 0 {
return address[:i]
}
return address
}
// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up
// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is
// down — the peers reach nothing.
func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string {
foundState, _ := sys.ServiceState(ctx, run, foundUnit)
meshState, _ := sys.ServiceState(ctx, run, meshUnit)
foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@"))
switch {
case meshState == "running" && !foundUp:
return Taken
case meshState == "running":
// Both up: not a takeover that holds, and said as not taken so nobody reads it as one.
return NotTaken
case foundUp:
return NotTaken
default:
return TunnelDown
}
}
// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the
// machine has the tunnel it had rather than none, and says so in the account.
func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) {
if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil {
facts.State = TunnelDown
facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit
return
}
if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" {
facts.State = TunnelDown
facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit
return
}
facts.State = NotTaken
facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
}
// retireFound removes the found tunnel's configuration from where its unit reads it, once the take
// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied
// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what
// replaces the take's outcome for the configuration.
//
// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's
// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up
// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it
// has checked its key, so that is the proof, asked of the kernel through `wg`. Anything short of
// one — no peer yet, every time zero, `wg` missing or failing — keeps the file, and the account
// says which: a take that never proves itself is visible rather than silently retired.
//
// **The original must still be kept.** It is the record of what the predecessor was and a
// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is
// not removed, since removing it then would lose the only copy. What is on disk now, if something
// other than the mesh rewrote it since it was found, is kept too before it goes — by content, so
// the first original is never overwritten.
//
// The found unit is left disabled; without its configuration it cannot raise the interface, so
// every later apply's check of it finds nothing to do. Nothing here ever writes the file back.
func retireFound(ctx context.Context, svc *declaration.Service, known *store.State, run Runner, keep Keep,
facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
t := svc.TakesOver
id := takeOverID(svc)
if facts.State != Taken {
return out, false
}
held, isHeld := known.HeldAt(id)
if !isHeld {
// Retired already (takeOver let any hold go and said so), or never held: nothing to do.
return out, false
}
say := func(note string) {
if facts.Note != "" {
facts.Note += "; "
}
facts.Note += note
}
mesh := strings.TrimPrefix(svc.Unit, "wg-quick@")
peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh)
if err != nil {
say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept")
return out, false
}
if peers == 0 {
say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " +
t.Config + " is kept")
return out, false
}
proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh)
// The kept original, read back — not just named in a record.
if held.Kept == "" {
say(proven + ", and the found configuration " + t.Config + " is not retired: no original of it " +
"was kept, so removing it would leave no record of what the predecessor was")
return out, false
}
original, err := os.ReadFile(held.Kept)
if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) {
why := "is missing"
if err == nil {
why = "no longer holds what was found"
} else if !errors.Is(err, os.ErrNotExist) {
why = "cannot be read (" + err.Error() + ")"
}
say(proven + ", and the found configuration " + t.Config + " is not retired: its kept original " +
held.Kept + " " + why + ", so removing it would lose the only copy")
return out, false
}
gone := !present(t.Config)
if !gone {
current, err := os.ReadFile(t.Config)
if err != nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: it cannot be read (" +
err.Error() + ")")
return out, false
}
if held.Digest != "" && digestOf(string(current)) != held.Digest {
if keep == nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: it was rewritten " +
"since it was found and this host has nowhere to keep what it holds now")
return out, false
}
if _, err := keep(t.Config, current, 0o600); err != nil {
say(proven + ", and the found configuration " + t.Config + " is not retired: keeping what it " +
"holds now failed (" + err.Error() + ")")
return out, false
}
}
if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) {
say(proven + ", and the found configuration " + t.Config + " could not be removed (" + err.Error() + ")")
return out, false
}
if present(t.Config) {
say(proven + ", and the found configuration " + t.Config + " is still there after it was removed")
return out, false
}
}
known.RecordRetired(store.Retired{ID: id, Path: t.Config, Kept: held.Kept, At: now})
known.Release(id)
facts.Kept = held.Kept
say(proven + "; the found configuration " + t.Config + " is retired — its original kept at " +
held.Kept + ", " + t.Unit + " left disabled, and the mesh never brings it back")
out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed",
Detail: "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept}
if gone {
// Already gone — removed by something other than the mesh, or by an apply whose record was
// never saved. Nothing removed here; the hold ends all the same.
out.Action = "unchanged"
out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config +
" was already gone; original kept at " + held.Kept
}
return out, true
}
// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
// a machine has one private network.
func takesOver(d *declaration.Declaration) *declaration.Service {
for _, r := range d.Resources {
if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil {
return svc
}
}
return nil
}
// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the
// parser refuses already; kept as a second line of defence at the point of acting.
var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only")