mesh-host/internal/apply/takeover.go

package apply

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/declaration"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
	"github.com/novox/mesh-host/internal/tunnel"
)

// The private network takes over the tunnel it found (novox/hq ADR 0105).
//
// The controller says so on the interface's service: `takes-over` names the found interface, the
// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps
// that file like any held file — the original recorded before anything else happens to it — and
// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the
// file is never written, and the found interface goes down the way its own unit takes it down.
// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found
// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands.
//
// Every apply, not once: a found unit somebody starts again would take the port back from the
// mesh's interface, so it is stopped again and said so. That is the one place an adopted node
// undoes something done by hand, and it is because the tunnel is the mesh's now.
//
// **Until the take is proven, and then the found configuration is retired** (novox/hq ADR 0119).
// Keeping it on disk was the caution the take needed: if the mesh's interface does not come up,
// the found unit is started again and the peers never notice. That caution is spent once the
// tunnel is taken — the found unit down and disabled, the mesh's interface up — and a peer has
// handshaken with the mesh's interface. From then on a configuration nothing maintains, one
// command away from raising a second way onto the network, is not a rollback path but a door
// nobody watches. So it is removed from where its unit reads it; its original, kept before
// anything happened to it (ADR 0100), stays kept; and the hold on it ends. A take never proven
// keeps it, and says so — a broken take is visible, not silently retired.

// TakenTunnel is what an apply says about a tunnel it took over, for the node's report.
type TakenTunnel struct {
	Interface string
	Port      int
	Range     string
	Peers     int
	// State is "not-taken" (the found interface still up, the mesh's not), "taken" (the found one
	// down and disabled, the mesh's up with its key) or "down" (the found one down and the mesh's
	// not up: the peers reach nothing). Note is what this apply did about it — and, for a taken
	// tunnel, whether the take is proven and its found configuration retired (novox/hq ADR 0119).
	// Kept is where the found configuration's original is, retired or not.
	State string
	Note  string
	Kept  string
}

// The states, as the link says them.
const (
	NotTaken   = "not-taken"
	Taken      = "taken"
	TunnelDown = "down"
)

// takeoverRecheck is how often, and takeoverRechecks how many times, a found interface still up
// after its unit stopped is looked at again before the takeover is refused: `wg-quick down` by a
// person takes a moment. Variables so a test need not wait.
var (
	takeoverRecheck  = 2 * time.Second
	takeoverRechecks = 3
)

// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
// so it is declared for as long as the service is and never mistaken for the service itself.
func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }

// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
// replaces it. Returned is the hold's outcome, and what was found for the report.
//
// **Nothing is stopped until the mesh's interface is known to be able to replace it** (the record's
// option 2 is exactly this going wrong): the declared configuration must listen on the found port
// at the found address, and the key file it points at must hold the found key. Only then is the
// found unit stopped — and `stopped` says whether this apply did, so a mesh interface that then
// fails to start can have the found unit started again.
func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
	known *store.State, run Runner, keep Keep, now time.Time) (out Outcome, facts TakenTunnel, stopped bool, err error) {
	t := svc.TakesOver
	id := takeOverID(svc)
	module, _ := d.Adoption.UntakenModuleOf(svc.ID)
	if module == "" {
		module = "the private network"
	}
	facts = TakenTunnel{Interface: t.Interface, State: NotTaken}

	// 0. What the found configuration says, before anything: the checks below are against it.
	found, ferr := readFoundTunnel(t.Config)

	// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
	//    the same code keeps its original, digests it and notices it changing.
	//
	//    **Unless it was retired** (novox/hq ADR 0119): the take was proven and the mesh removed
	//    it, so there is nothing to hold and nothing missing — only where its original is, which
	//    the retirement recorded. A hold still standing is let go: that is what retiring it meant.
	//
	//    **One that comes back is held again on its FIRST original** — the one kept before anything
	//    happened to it (ADR 0100), never whatever was put back — and retired again by the first
	//    apply that finds the take still proven, keeping what came back only if it differs from
	//    what is already kept. Put back by hand while the private network is assigned, it is not a
	//    rollback: that means unassigning the private network first, and the note says so.
	file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
	var held store.Held
	retired, wasRetired := known.RetiredAt(t.Config)
	cameBack := wasRetired && present(t.Config)
	switch {
	case wasRetired && !cameBack:
		known.Release(id)
		held = store.Held{Kept: retired.Kept}
		out = begin(file)
		out.Action = "unchanged"
		facts.Note = "the found configuration " + t.Config + " was retired once the take was proven; " +
			"its original is kept at " + retired.Kept + " and the mesh never brings it back"
	default:
		was, already := known.HeldAt(id)
		why := "the configuration of the tunnel " + t.Interface + ", taken over by " + svc.Unit
		if cameBack && !already {
			digest := retired.Digest
			if digest == "" {
				if raw, err := os.ReadFile(retired.Kept); err == nil {
					digest = digestOf(string(raw))
				}
			}
			was = store.Held{ID: id, Module: module, Kind: string(declaration.TypeFile), Target: t.Config,
				Since: now, Why: why, Kept: retired.Kept, Digest: digest}
			already = true
		}
		out, held, err = hold(ctx, sys, file, module, was, already, why, run, keep, now)
		if err != nil {
			return begin(file), facts, false, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
		}
		known.RecordHeld(held)
		if cameBack {
			facts.Note = "the found configuration " + t.Config + " came back after it was retired; while the " +
				"private network is assigned the mesh retires it again, so rolling back to the found tunnel " +
				"means unassigning the private network first"
		}
	}
	facts.Kept = held.Kept
	// What the file says, for the report: from the machine, or from the kept original when the
	// machine's copy is gone. The private key stays in the file; nothing here keeps it.
	unread := ""
	if ferr != nil && held.Kept != "" {
		found, ferr = readFoundTunnel(held.Kept)
	}
	if ferr == nil {
		facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
	} else {
		unread = ferr.Error()
	}

	// 2. Where things stand: the found unit, and the mesh's.
	foundState, unitErr := sys.ServiceState(ctx, run, t.Unit)
	meshState, _ := sys.ServiceState(ctx, run, svc.Unit)
	if foundState == "running" && meshState == "running" {
		// Both up. On the hub this cannot last — the found unit cannot bind the port the mesh's
		// holds — and on a spoke two interfaces with one key flap between them. Not stopped again
		// by the mesh: what is found on an adopted node is reported, and the first takeover was
		// the one act (the PR note says why). Said, so a person sees it.
		facts.Note = t.Unit + " is running again beside the mesh's interface; not stopped by the mesh — " +
			"`systemctl stop " + t.Unit + "` on the machine"
	}

	// 3. Before the found unit is stopped: can the mesh's interface replace it? Its declared
	//    configuration must listen on the found port at the found address, and the key file it
	//    points at must hold the found key, or the peers would be dropped the moment it came up.
	if foundState == "running" && meshState != "running" {
		if ferr != nil {
			return out, facts, false, fmt.Errorf("the found tunnel's configuration at %s cannot be read as a "+
				"tunnel's (%v), so nothing says what the mesh's interface must match; %s is left running",
				t.Config, ferr, t.Unit)
		}
		if err := replaces(d, svc, found); err != nil {
			return out, facts, false, fmt.Errorf("%w; %s is left running", err, t.Unit)
		}
	}

	// 4. The found unit: stopped if it runs and the mesh's does not, disabled if it starts at
	//    boot. A unit that is not there is not an error — the interface may have been raised
	//    another way, which the check below catches — and neither is one already down.
	var did []string
	switch {
	case unitErr != nil:
		did = append(did, t.Unit+" is not a unit here")
	case foundState == "running" && meshState != "running":
		if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
			return out, facts, false, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
		}
		after, err := sys.ServiceState(ctx, run, t.Unit)
		if err != nil {
			return out, facts, true, err
		}
		if after != "stopped" {
			return out, facts, true, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
		}
		stopped = true
		did = append(did, "stopped "+t.Unit)
	}
	if unitErr == nil {
		if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
			if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
				return out, facts, stopped, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
			}
			did = append(did, "disabled it at boot")
		}
	}

	// 5. The interface is gone. If it is still up, something other than its unit raised it —
	//    the predecessor brings its up by hand — and the mesh's interface cannot take its port
	//    and address while it is. Looked at again for a moment, since a person taking it down
	//    takes a moment; then refused, naming what to do.
	if meshState != "running" {
		for try := 0; ; try++ {
			if !interfaceUp(ctx, run, t.Interface) {
				break
			}
			if try >= takeoverRechecks {
				return out, facts, stopped, fmt.Errorf("%s is still up although its unit %s is not running: it was "+
					"raised by hand, not by its unit, and the mesh's interface cannot take its port and "+
					"address while it is. On the machine: `wg-quick down %s` — the next reconcile takes it "+
					"over. Nothing was flushed", t.Interface, t.Unit, t.Interface)
			}
			select {
			case <-ctx.Done():
				return out, facts, stopped, ctx.Err()
			case <-time.After(takeoverRecheck):
			}
		}
	}

	out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
	switch {
	case cameBack:
		out.Detail = "the tunnel " + t.Interface + "'s configuration, back after it was retired; held until " +
			"it is retired again"
	case wasRetired:
		out.Detail = "the tunnel " + t.Interface + "'s configuration, retired once the take was proven"
	}
	if held.Kept != "" {
		out.Detail += " (original at " + held.Kept + ")"
	}
	if len(did) > 0 {
		out.Detail += "; " + strings.Join(did, ", ") + " — never flushed"
	}
	if held.Changed != "" {
		out.Detail += "; " + held.Changed + " by something other than the mesh since it was found"
	}
	if unread != "" {
		// Said, not swallowed: the report would otherwise say a tunnel with no port and no
		// peers was carried, which reads as a tunnel that was not one.
		out.Detail += "; what it says could not be read as a tunnel's: " + unread
	}
	return out, facts, stopped, nil
}

// readFoundTunnel is the found configuration as a tunnel.
func readFoundTunnel(path string) (tunnel.Found, error) {
	raw, err := os.ReadFile(path)
	if err != nil {
		return tunnel.Found{}, err
	}
	return tunnel.Parse(raw)
}

// interfaceUp is whether a WireGuard interface is up on the machine.
func interfaceUp(ctx context.Context, run Runner, iface string) bool {
	up, err := run(ctx, "wg", "show", "interfaces")
	if err != nil {
		return false
	}
	for _, name := range strings.Fields(up) {
		if name == iface {
			return true
		}
	}
	return false
}

// replaces holds the mesh's declared interface configuration against the found tunnel it is to
// replace: same port, same address, and a key file holding the found key. The configuration is
// the file the service restarts on; its `PostUp = wg set %i private-key <path>` names the key.
func replaces(d *declaration.Declaration, svc *declaration.Service, found tunnel.Found) error {
	var conf *declaration.File
	for _, r := range d.Resources {
		f, ok := r.(*declaration.File)
		if !ok {
			continue
		}
		for _, id := range svc.RestartOn {
			if f.ID == id {
				conf = f
			}
		}
	}
	if conf == nil {
		return fmt.Errorf("%s takes over %s and restarts on no declared file, so the interface it would "+
			"raise cannot be checked against the found one", svc.Unit, found.Interface)
	}
	port, address, keyPath := "", "", ""
	for _, line := range strings.Split(conf.Content, "\n") {
		key, value, ok := strings.Cut(strings.TrimSpace(line), "=")
		if !ok {
			continue
		}
		key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value)
		switch key {
		case "listenport":
			port = value
		case "address":
			address = strings.TrimSpace(strings.Split(value, ",")[0])
		case "postup":
			if _, after, ok := strings.Cut(value, "private-key "); ok {
				keyPath = strings.Fields(after)[0]
			}
		}
	}
	var wrong []string
	if port != fmt.Sprint(found.Port) {
		wrong = append(wrong, fmt.Sprintf("it listens on port %q and the tunnel on %d", port, found.Port))
	}
	if host(address) != host(found.Address) {
		wrong = append(wrong, fmt.Sprintf("its address is %q and the tunnel's %s", address, found.Address))
	}
	switch raw, err := os.ReadFile(keyPath); {
	case keyPath == "":
		wrong = append(wrong, "it names no key file")
	case err != nil:
		wrong = append(wrong, fmt.Sprintf("its key file %s cannot be read (%v)", keyPath, err))
	default:
		public, perr := tunnel.PublicKeyOf(strings.TrimSpace(string(raw)))
		if perr != nil || public != found.PublicKey {
			wrong = append(wrong, fmt.Sprintf("the key at %s is not the tunnel's — `mesh-host overlay take "+
				"--tunnel %s` on this machine takes it, then push again", keyPath, found.Interface))
		}
	}
	if len(wrong) > 0 {
		return fmt.Errorf("the mesh's interface would not replace the tunnel on %s: %s — the peers would be "+
			"dropped the moment it came up. Re-place the hub on the tunnel's address and port and push again",
			found.Interface, strings.Join(wrong, "; "))
	}
	return nil
}

// host is an address without its prefix length.
func host(address string) string {
	if i := strings.Index(address, "/"); i >= 0 {
		return address[:i]
	}
	return address
}

// tunnelState is where the tunnel stands, read from the machine: the found unit or interface up
// and the mesh's not is not taken; the mesh's up and the found one down is taken; neither up is
// down — the peers reach nothing.
func tunnelState(ctx context.Context, sys system.System, foundUnit, meshUnit string, run Runner) string {
	foundState, _ := sys.ServiceState(ctx, run, foundUnit)
	meshState, _ := sys.ServiceState(ctx, run, meshUnit)
	foundUp := foundState == "running" || interfaceUp(ctx, run, strings.TrimPrefix(foundUnit, "wg-quick@"))
	switch {
	case meshState == "running" && !foundUp:
		return Taken
	case meshState == "running":
		// Both up: not a takeover that holds, and said as not taken so nobody reads it as one.
		return NotTaken
	case foundUp:
		return NotTaken
	default:
		return TunnelDown
	}
}

// restoreFound starts the found unit again after the mesh's interface failed to replace it, so the
// machine has the tunnel it had rather than none, and says so in the account.
func restoreFound(ctx context.Context, sys system.System, unit string, run Runner, facts *TakenTunnel) {
	if err := sys.SetServiceState(ctx, run, unit, "running"); err != nil {
		facts.State = TunnelDown
		facts.Note += "; " + unit + " could not be started again (" + err.Error() + ") — on the machine: systemctl start " + unit
		return
	}
	if state, err := sys.ServiceState(ctx, run, unit); err != nil || state != "running" {
		facts.State = TunnelDown
		facts.Note += "; " + unit + " was started again and is not running — on the machine: systemctl start " + unit
		return
	}
	facts.State = NotTaken
	facts.Note += "; " + unit + " was started again, so the machine has the tunnel it had"
}

// wireguardDir is where a found tunnel's configuration may be retired from: wg-quick's own, and
// nowhere else. A variable so a test can hand in a directory.
var wireguardDir = tunnel.ConfigDir

// retireFound removes the found tunnel's configuration from where its unit reads it, once the take
// is proven, and ends the hold on it (novox/hq ADR 0119). Asked after the mesh's service applied
// and the tunnel reads as taken; retired says whether this apply retired it, and out is then what
// replaces the take's outcome for the configuration.
//
// **Proven is taken and a handshake.** Taken alone — the found unit down and disabled, the mesh's
// interface up — says the mesh's interface exists, not that any peer reaches it: an interface up
// with the wrong key is taken and carries nothing. A peer that has completed a handshake with it
// has checked its key, so that is the proof, asked of the kernel through `wg`. Any handshake counts,
// however old: a change to the mesh's configuration restarts its unit, which recreates the
// interface and resets its counters, so a time that is there at all was made by this interface.
// Anything short of one — no peer yet, every time zero, `wg` missing or failing — keeps the file,
// and the account says which: a take that never proves itself is visible rather than silently
// retired.
//
// **Only what the take names, and only wg-quick's own file.** Nothing is removed unless the path
// is exactly `<wireguard dir>/<found interface>.conf`, is not a path the mesh itself writes, and is
// a file rather than a link: removing a link would leave the key-bearing file it points at where it
// is, a retirement in name only, so that one is said and left to a person.
//
// **The original must still be kept.** It is the record of what the predecessor was and a
// person's only way back (ADR 0100); a kept copy that has gone missing is said, and the file is
// not removed, since removing it then would lose the only copy. What is on disk now, if it differs
// from the first original and from what was kept at the last retirement, is kept too before it
// goes — by content, so the first original is never overwritten and a file that keeps coming back
// the same keeps nothing more.
//
// The found unit is left disabled; without its configuration it cannot raise the interface, so
// every later apply's check of it finds nothing to do. Nothing here ever writes the file back.
func retireFound(ctx context.Context, svc *declaration.Service, d *declaration.Declaration, known *store.State,
	run Runner, keep Keep, facts *TakenTunnel, now time.Time) (out Outcome, retired bool) {
	t := svc.TakesOver
	id := takeOverID(svc)
	if facts.State != Taken {
		return out, false
	}
	held, isHeld := known.HeldAt(id)
	if !isHeld {
		// Retired already (takeOver let any hold go and said so), or never held: nothing to do.
		return out, false
	}
	say := func(note string) {
		if facts.Note != "" {
			facts.Note += "; "
		}
		facts.Note += note
	}
	notRetired := func(why string) (Outcome, bool) {
		say("the found configuration " + t.Config + " is not retired: " + why)
		return Outcome{}, false
	}
	mesh := strings.TrimPrefix(svc.Unit, "wg-quick@")
	peers, err := tunnel.Handshaken(ctx, tunnel.Runner(run), mesh)
	if err != nil {
		say("taken, not yet proven: " + err.Error() + "; the found configuration " + t.Config + " is kept")
		return out, false
	}
	if peers == 0 {
		say("taken, not yet proven: no peer has handshaken on " + mesh + "; the found configuration " +
			t.Config + " is kept")
		return out, false
	}
	proven := fmt.Sprintf("proven: %d peer(s) handshaken on %s", peers, mesh)
	say(proven)

	// What may be removed at all.
	if want := filepath.Join(wireguardDir, t.Interface+".conf"); t.Config != want {
		return notRetired("only " + want + ", the found interface's own wg-quick configuration, is ever " +
			"retired by the mesh, and the take names " + t.Config)
	}
	if known.Recorded(string(declaration.TypeFile), t.Config) || declaresFile(d, t.Config) {
		return notRetired("it is a path the mesh itself writes")
	}
	if info, err := os.Lstat(t.Config); err == nil && info.Mode()&os.ModeSymlink != 0 {
		target, _ := os.Readlink(t.Config)
		return notRetired("it is a link to " + target + "; removing the link would leave the key-bearing file " +
			"it points at, so it must be retired by hand — both are kept")
	}

	// The kept original, read back — not just named in a record.
	if held.Kept == "" {
		return notRetired("no original of it was kept, so removing it would leave no record of what the " +
			"predecessor was")
	}
	original, err := os.ReadFile(held.Kept)
	if err != nil || (held.Digest != "" && digestOf(string(original)) != held.Digest) {
		why := "is missing"
		if err == nil {
			why = "no longer holds what was found"
		} else if !errors.Is(err, os.ErrNotExist) {
			why = "cannot be read (" + err.Error() + ")"
		}
		return notRetired("its kept original " + held.Kept + " " + why + ", so removing it would lose the only copy")
	}

	before, cameBack := known.RetiredAt(t.Config)
	record := store.Retired{ID: id, Path: t.Config, Kept: held.Kept, Digest: digestOf(string(original)), At: now}
	if cameBack {
		// The first original stays the record's, and so does what the last retirement kept.
		record.Extra, record.ExtraDigest, record.Again = before.Extra, before.ExtraDigest, before.Again+1
	}
	newCopy := ""
	gone := !present(t.Config)
	if !gone {
		current, err := os.ReadFile(t.Config)
		if err != nil {
			return notRetired("it cannot be read (" + err.Error() + ")")
		}
		if sum := digestOf(string(current)); sum != record.Digest && sum != record.ExtraDigest {
			if keep == nil {
				return notRetired("it holds something other than its kept original and this host has nowhere " +
					"to keep it")
			}
			where, err := keep(t.Config, current, 0o600)
			if err != nil {
				return notRetired("keeping what it holds now failed (" + err.Error() + ")")
			}
			record.Extra, record.ExtraDigest, newCopy = where, sum, where
		}
		if err := os.Remove(t.Config); err != nil && !errors.Is(err, os.ErrNotExist) {
			return notRetired("removing it failed (" + err.Error() + ")")
		}
		if present(t.Config) {
			return notRetired("it is still there after it was removed")
		}
	}

	known.RecordRetired(record)
	known.Release(id)
	facts.Kept = held.Kept
	copied := ""
	if newCopy != "" {
		copied = "; what it held, which differed from the original, is kept at " + newCopy
	}
	out = Outcome{ID: id, Type: string(declaration.TypeFile), Target: t.Config, Action: "removed"}
	switch {
	case cameBack:
		say("the found configuration came back and was retired again — its original still kept at " +
			held.Kept + copied + "; rolling back to the found tunnel means unassigning the private network first")
		out.Detail = "the found configuration came back and was retired again; original kept at " + held.Kept + copied
	default:
		say("the found configuration " + t.Config + " is retired — its original kept at " + held.Kept + copied +
			", " + t.Unit + " left disabled, and the mesh never brings it back")
		out.Detail = "retired: the take of " + t.Interface + " is " + proven + "; original kept at " + held.Kept + copied
	}
	if gone {
		// Already gone — removed by something other than the mesh, or by an apply whose record was
		// never saved. Nothing removed here; the hold ends all the same.
		out.Action = "unchanged"
		out.Detail = "retired: the take of " + t.Interface + " is " + proven + " and " + t.Config +
			" was already gone; original kept at " + held.Kept
	}
	return out, true
}

// declaresFile is whether a declaration writes a file at a path.
func declaresFile(d *declaration.Declaration, path string) bool {
	for _, r := range d.Resources {
		if f, ok := r.(*declaration.File); ok && filepath.Clean(f.Path) == filepath.Clean(path) {
			return true
		}
	}
	return false
}

// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
// a machine has one private network.
func takesOver(d *declaration.Declaration) *declaration.Service {
	for _, r := range d.Resources {
		if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil {
			return svc
		}
	}
	return nil
}

// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the
// parser refuses already; kept as a second line of defence at the point of acting.
var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only")