Take over the found tunnel: its key, its port, its peers; stop it, never flush

On an adopted machine the private network takes the predecessor's tunnel
over in place (hq ADR 0105). Genesis finds the one interface up besides the
mesh's own, settles the hub's port and the mesh's range on it, and skips
ADR 0100's non-overlap check for a range that is now the tunnel's; a
--hub-port or --overlay-range that disagrees is refused naming the tunnel's.

At enrolment the found interface's private key becomes this node's overlay
key — the one credential the mesh takes rather than mints — stored where a
generated one is stored, never printed and never sent; the tunnel (port,
address, range, peers) travels with the keys so the mesh composes from it
before the first declaration.

The interface's service may say what it takes over. Before the mesh's unit
starts, the found configuration is kept like any held file and the found
unit is stopped and disabled; nothing is flushed, and an interface still up
after its unit stopped refuses the takeover rather than half-working. The
report says what was carried: interface, port, range, peer count, taken or
not, and where the original was kept.

internal/apply/apply.go

@@ -57,6 +57,9 @@ type Outcome struct {
 // Report is what an apply did, in the order it did it.
 type Report struct {
 	Outcomes []Outcome `json:"outcomes"`
+	// Tunnel is what this apply says about the tunnel the private network took over, when the
+	// declaration names one (novox/hq ADR 0105).
+	Tunnel *TakenTunnel `json:"tunnel,omitempty"`
 }
 
 // Changed reports whether anything about the machine actually moved. An apply that changed
@@ -153,6 +156,11 @@ func ApplyKeeping(
 	for _, r := range d.Resources {
 		declared[r.Identity()] = true
 	}
+	if svc := takesOver(d); svc != nil {
+		// The found tunnel's configuration is held under an id of its own, declared for as long
+		// as the service that took it over is (novox/hq ADR 0105).
+		declared[takeOverID(svc)] = true
+	}
 
 	// Which firewall is found here, before anything else, since an unsupported one refuses the
 	// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
@@ -327,6 +335,29 @@ func ApplyKeeping(
 			}
 		}
 
+		// The private network takes over the tunnel it found, ahead of the service that replaces
+		// it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never
+		// flushed. A failure here fails the service too — the mesh's interface is not started on a
+		// port the found one still holds.
+		if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil {
+			var outcome Outcome
+			var facts TakenTunnel
+			var err error
+			if d.Adoption == nil {
+				err = errNotAdopted
+			} else {
+				outcome, facts, err = takeOver(ctx, sys, svc, d, &known, run, keep, time.Now().UTC())
+			}
+			if err != nil {
+				failures = append(failures, &Error{Resource: svc.Identity(), Err: err, Done: report})
+				log(fmt.Sprintf("  failed %s (%s): %v", svc.Identity(), svc.Unit, err))
+				continue
+			}
+			report.Outcomes = append(report.Outcomes, outcome)
+			report.Tunnel = &facts
+			log(fmt.Sprintf("  held %s (%s): %s", outcome.ID, outcome.Target, outcome.Detail))
+		}
+
 		was, _ := known.Find(resource.Identity())
 		var outcome Outcome
 		var err error
@@ -393,6 +424,11 @@ func ApplyKeeping(
 			known.Release(held.ID)
 			outcome.Detail = takenDetail(held)
 		}
+		if svc, ok := resource.(*declaration.Service); ok && svc.TakesOver != nil && report.Tunnel != nil {
+			// The found interface is down and the mesh's is up in its place: the tunnel changed
+			// hands (novox/hq ADR 0105).
+			report.Tunnel.Taken = true
+		}
 		report.Outcomes = append(report.Outcomes, outcome)
 		if outcome.Action != "unchanged" {
 			changed[resource.Identity()] = true

internal/apply/hold_test.go

@@ -19,6 +19,8 @@ import (
 type machine struct {
 	containers map[string]*fakeContainer
 	asked      []string
+	// wgUp is what `wg show interfaces` answers: the tunnels up on the machine.
+	wgUp string
 
 	// units are service units by name, as systemd would report them; volumes are the runtime's
 	// named volumes.
@@ -94,6 +96,9 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
 	if name == "systemctl" {
 		return m.systemctl(args)
 	}
+	if name == "wg" {
+		return m.wgUp, nil
+	}
 	if name == "getent" {
 		if m.users[args[len(args)-1]] {
 			return args[len(args)-1] + ":x:1500:1500::/home/" + args[len(args)-1] + ":/bin/bash\n", nil

internal/apply/takeover.go

@@ -0,0 +1,159 @@
+package apply
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+	"github.com/novox/mesh-host/internal/system"
+	"github.com/novox/mesh-host/internal/tunnel"
+)
+
+// The private network takes over the tunnel it found (novox/hq ADR 0105).
+//
+// The controller says so on the interface's service: `takes-over` names the found interface, the
+// unit that raised it and its configuration file. Before the mesh's unit is started, the host keeps
+// that file like any held file — the original recorded before anything else happens to it — and
+// stops and disables the found unit. Never a flush: `wg set … peer … remove` is never run, the
+// file is never written, and the found interface goes down the way its own unit takes it down.
+// Then the mesh's interface comes up, with the found key the node took at enrolment, on the found
+// port, with the found peers in its list — and a peer of the tunnel cannot tell it changed hands.
+//
+// Every apply, not once: a found unit somebody starts again would take the port back from the
+// mesh's interface, so it is stopped again and said so. That is the one place an adopted node
+// undoes something done by hand, and it is because the tunnel is the mesh's now.
+
+// TakenTunnel is what an apply says about a tunnel it took over, for the node's report.
+type TakenTunnel struct {
+	Interface string
+	Port      int
+	Range     string
+	Peers     int
+	// Taken is whether the found interface is down and disabled and the mesh's up in its place.
+	Taken bool
+	Kept  string
+}
+
+// takeOverID is the held record's id for the found configuration: the service's own with a suffix,
+// so it is declared for as long as the service is and never mistaken for the service itself.
+func takeOverID(svc *declaration.Service) string { return svc.ID + ".takes-over" }
+
+// takeOver keeps the found tunnel's configuration and stops its unit, ahead of the service that
+// replaces it. Returned is the hold's outcome, and what was found for the report.
+func takeOver(ctx context.Context, sys system.System, svc *declaration.Service, d *declaration.Declaration,
+	known *store.State, run Runner, keep Keep, now time.Time) (Outcome, TakenTunnel, error) {
+	t := svc.TakesOver
+	id := takeOverID(svc)
+	module, _ := d.Adoption.UntakenModuleOf(svc.ID)
+	if module == "" {
+		module = "the private network"
+	}
+	facts := TakenTunnel{Interface: t.Interface}
+
+	// 1. The configuration, kept like any held file. A synthetic file resource stands for it, so
+	//    the same code keeps its original, digests it and notices it changing.
+	file := &declaration.File{ID: id, Type: declaration.TypeFile, Path: t.Config}
+	was, already := known.HeldAt(id)
+	out, held, err := hold(ctx, sys, file, module, was, already,
+		"the configuration of the tunnel "+t.Interface+", taken over by "+svc.Unit, run, keep, now)
+	if err != nil {
+		return begin(file), facts, fmt.Errorf("keeping the found tunnel's configuration: %w", err)
+	}
+	known.RecordHeld(held)
+	facts.Kept = held.Kept
+	// What the file says, for the report: read from the machine, or from the kept original when
+	// the machine's copy is gone. The private key stays in the file; nothing here keeps it.
+	unread := ""
+	raw, err := os.ReadFile(t.Config)
+	if err != nil && held.Kept != "" {
+		raw, err = os.ReadFile(held.Kept)
+	}
+	if err == nil {
+		if found, perr := tunnel.Parse(raw); perr == nil {
+			facts.Port, facts.Range, facts.Peers = found.Port, found.Range, len(found.Peers)
+		} else {
+			unread = perr.Error()
+		}
+	} else {
+		unread = err.Error()
+	}
+
+	// 2. The found unit: stopped if it runs, disabled if it starts at boot. A unit that is not
+	//    there is not an error — the interface may have been raised another way, which the check
+	//    below catches — and neither is one already down.
+	var did []string
+	state, err := sys.ServiceState(ctx, run, t.Unit)
+	switch {
+	case err != nil:
+		did = append(did, t.Unit+" is not a unit here")
+	case state == "running":
+		if err := sys.SetServiceState(ctx, run, t.Unit, "stopped"); err != nil {
+			return out, facts, fmt.Errorf("stopping the found %s: %w", t.Unit, err)
+		}
+		after, err := sys.ServiceState(ctx, run, t.Unit)
+		if err != nil {
+			return out, facts, err
+		}
+		if after != "stopped" {
+			return out, facts, fmt.Errorf("%s was asked to stop and is %s", t.Unit, after)
+		}
+		did = append(did, "stopped "+t.Unit)
+	}
+	if err == nil {
+		if boot, err := sys.ServiceBoot(ctx, run, t.Unit); err == nil && boot == "enabled" {
+			if err := sys.SetServiceBoot(ctx, run, t.Unit, "disabled"); err != nil {
+				return out, facts, fmt.Errorf("disabling the found %s at boot: %w", t.Unit, err)
+			}
+			did = append(did, "disabled it at boot")
+		}
+	}
+
+	// 3. The interface is gone. If it is still up, something other than its unit raised it, and
+	//    starting the mesh's on the same port and address would fail or, worse, half work.
+	if up, err := run(ctx, "wg", "show", "interfaces"); err == nil {
+		for _, iface := range strings.Fields(up) {
+			if iface == t.Interface {
+				return out, facts, fmt.Errorf("%s is still up after its unit %s was stopped: something other "+
+					"than that unit raises it, and the mesh's interface cannot take its port and address "+
+					"while it does. Nothing was flushed", t.Interface, t.Unit)
+			}
+		}
+	}
+
+	out.Detail = "the tunnel " + t.Interface + "'s configuration, kept as found"
+	if held.Kept != "" {
+		out.Detail += " (original at " + held.Kept + ")"
+	}
+	if len(did) > 0 {
+		out.Detail += "; " + strings.Join(did, ", ") + " — never flushed"
+	}
+	if held.Changed != "" {
+		out.Detail += "; " + held.Changed + " by something other than the mesh since it was found"
+	}
+	if unread != "" {
+		// Said, not swallowed: the report would otherwise say a tunnel with no port and no
+		// peers was carried, which reads as a tunnel that was not one.
+		out.Detail += "; what it says could not be read as a tunnel's: " + unread
+	}
+	return out, facts, nil
+}
+
+// takesOver is the service in a declaration that takes over a tunnel, if any: one per node, since
+// a machine has one private network.
+func takesOver(d *declaration.Declaration) *declaration.Service {
+	for _, r := range d.Resources {
+		if svc, ok := r.(*declaration.Service); ok && svc.TakesOver != nil {
+			return svc
+		}
+	}
+	return nil
+}
+
+// errNotAdopted is a takeover on a declaration that does not say the node is adopted, which the
+// parser refuses already; kept as a second line of defence at the point of acting.
+var errNotAdopted = errors.New("a tunnel is taken over on an adopted node only")

internal/apply/takeover_test.go

@@ -0,0 +1,165 @@
+package apply
+
+import (
+	"crypto/ecdh"
+	"crypto/rand"
+	"encoding/base64"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-host/internal/declaration"
+	"github.com/novox/mesh-host/internal/store"
+)
+
+// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
+// found interface without flushing it, and keeps its configuration.
+
+// foundConf is the predecessor's configuration, with a real key made once per run: the key is
+// what the takeover must never print or copy, so it had better be one.
+var foundConf = func() string {
+	k, err := ecdh.X25519().GenerateKey(rand.Reader)
+	if err != nil {
+		panic(err)
+	}
+	return "[Interface]\nPrivateKey = " + base64.StdEncoding.EncodeToString(k.Bytes()) + "\n" +
+		"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
+		"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"
+}()
+
+// aTakeover is the private network's declaration for an adopted hub whose interface takes over
+// the found tunnel: the mesh's configuration — with the found key set from the node's own key file
+// and the found peers in its list — and the interface's service naming what it replaces.
+func aTakeover(t *testing.T, config, mesh string) *declaration.Declaration {
+	t.Helper()
+	return adopted(t,
+		`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
+		`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
+		  "content":"[Interface]\nAddress = 192.0.2.1/32\nListenPort = 51900\nPostUp = wg set %i private-key /var/lib/mesh-host/overlay.key\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
+		 {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
+		  "restart-on":["mesh-wireguard.overlay-config"],
+		  "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
+}
+
+// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet.
+func aHubInUse(t *testing.T) (dir, config, mesh string, m *machine) {
+	t.Helper()
+	dir = t.TempDir()
+	config = filepath.Join(dir, "wg0.conf")
+	mesh = filepath.Join(dir, "mesh0.conf")
+	if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
+		"wg-quick@wg0":   {active: "active", enabled: "enabled"},
+		"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
+	}}
+	return dir, config, mesh, m
+}
+
+func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
+	dir, config, mesh, m := aHubInUse(t)
+	report, state := applyAdopted(t, aTakeover(t, config, mesh), store.State{}, m, dir)
+
+	// The found interface: its unit stopped and disabled, and nothing else done to it.
+	if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
+		t.Fatalf("the found unit was not stopped and disabled: %+v", u)
+	}
+	for _, asked := range m.asked {
+		if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") {
+			t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked)
+		}
+		if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") {
+			t.Errorf("the found interface was flushed: %q", asked)
+		}
+	}
+	// Its configuration: on disk as it was, its original kept, held for the module.
+	if got, _ := os.ReadFile(config); string(got) != foundConf {
+		t.Fatalf("the found configuration was changed:\n%s", got)
+	}
+	held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over")
+	if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" {
+		t.Fatalf("the found configuration is not held: %+v", held)
+	}
+	if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
+		t.Fatalf("the original was not kept as found: %q", kept)
+	}
+	// The mesh's interface: up, enabled, with the found peers in the file the mesh wrote.
+	if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" {
+		t.Fatalf("the mesh's interface was not raised: %+v", u)
+	}
+	if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") {
+		t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
+	}
+	// And the report says so, with what was found — port, range, peers — and never the key.
+	if report.Tunnel == nil || !report.Tunnel.Taken || report.Tunnel.Port != 51900 ||
+		report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
+		t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
+	}
+	private := strings.TrimSpace(strings.SplitN(strings.SplitN(foundConf, "PrivateKey = ", 2)[1], "\n", 2)[0])
+	for _, o := range report.Outcomes {
+		if strings.Contains(o.Detail, private) {
+			t.Errorf("the found key was printed in an outcome: %+v", o)
+		}
+	}
+	if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
+		!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
+		t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
+	}
+	if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded {
+		t.Error("the found configuration was recorded as applied, so it would be removed as an orphan")
+	}
+}
+
+func TestATakeoverIsSteadyAndTheFoundUnitStaysDown(t *testing.T) {
+	dir, config, mesh, m := aHubInUse(t)
+	d := aTakeover(t, config, mesh)
+	_, state := applyAdopted(t, d, store.State{}, m, dir)
+	m.asked = nil
+
+	report, again := applyAdopted(t, d, state, m, dir)
+	if report.Changed() {
+		t.Errorf("a second apply moved the machine: %+v", report.Outcomes)
+	}
+	if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still {
+		t.Error("the hold on the found configuration was forgotten while the service still declares it")
+	}
+	if m.did("systemctl stop wg-quick@wg0") {
+		t.Error("a found unit already down was stopped again")
+	}
+
+	// Somebody starts the found unit again: it would take the port back, so it is stopped again
+	// — the one thing on an adopted node the mesh undoes, because the tunnel is the mesh's now.
+	m.units["wg-quick@wg0"].active = "active"
+	m.asked = nil
+	_, _ = applyAdopted(t, d, again, m, dir)
+	if m.units["wg-quick@wg0"].active != "inactive" || !m.did("systemctl stop wg-quick@wg0") {
+		t.Error("a found unit started again was left holding the mesh's port")
+	}
+}
+
+func TestAFoundInterfaceStillUpAfterItsUnitStoppedRefusesTheTakeover(t *testing.T) {
+	dir, config, mesh, m := aHubInUse(t)
+	m.wgUp = "wg0 mesh0\n"
+	_, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh), store.State{},
+		store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
+	if err == nil || !strings.Contains(err.Error(), "still up") || !strings.Contains(err.Error(), "Nothing was flushed") {
+		t.Fatalf("an interface something else raises was taken over anyway: %v", err)
+	}
+	if m.units["wg-quick@mesh0"].active == "active" {
+		t.Error("the mesh's interface was started on a port the found one still holds")
+	}
+	if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
+		t.Error("the found configuration was not kept before the refusal")
+	}
+}
+
+func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) {
+	_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
+	  {"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
+	   "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`))
+	if err == nil || !strings.Contains(err.Error(), "adopted") {
+		t.Fatalf("a takeover on a converged node was accepted: %v", err)
+	}
+}