mesh-host/internal/apply/takeover_test.go

package apply

import (
	"crypto/ecdh"
	"crypto/rand"
	"encoding/base64"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
// found interface without flushing it, and keeps its configuration — and stops nothing until the
// mesh's interface is known to be able to replace it.

// foundKey is the predecessor's private key, a real one made once per run: the key is what the
// takeover must never print or copy, so it had better be one.
var foundKey = func() string {
	k, err := ecdh.X25519().GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	return base64.StdEncoding.EncodeToString(k.Bytes())
}()

var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" +
	"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
	"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"

// aTakeover is the private network's declaration for an adopted hub whose interface takes over
// the found tunnel: the mesh's configuration — on the found port and address, its key set from the
// node's own key file, the found peers in its list — and the interface's service naming what it
// replaces. Port and address are parameters so a test can declare a wrong one.
func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration {
	t.Helper()
	return adopted(t,
		`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
		`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
		  "content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
		 {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
		  "restart-on":["mesh-wireguard.overlay-config"],
		  "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
}

// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found
// configuration on disk, and the node's key file holding the found key, as enrolment left it.
func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
	t.Helper()
	dir = t.TempDir()
	config = filepath.Join(dir, "wg0.conf")
	mesh = filepath.Join(dir, "mesh0.conf")
	keyFile = filepath.Join(dir, "overlay.key")
	if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
		"wg-quick@wg0":   {active: "active", enabled: "enabled"},
		"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
	}}
	takeoverRecheck = 0
	return dir, config, mesh, keyFile, m
}

func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)

	// The found interface: its unit stopped and disabled, and nothing else done to it.
	if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Fatalf("the found unit was not stopped and disabled: %+v", u)
	}
	for _, asked := range m.asked {
		if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") {
			t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked)
		}
		if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") {
			t.Errorf("the found interface was flushed: %q", asked)
		}
	}
	// Its configuration: on disk as it was, its original kept, held for the module.
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatalf("the found configuration was changed:\n%s", got)
	}
	held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over")
	if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" {
		t.Fatalf("the found configuration is not held: %+v", held)
	}
	if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
		t.Fatalf("the original was not kept as found: %q", kept)
	}
	// The mesh's interface: up, enabled, with the found peers in the file the mesh wrote.
	if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" {
		t.Fatalf("the mesh's interface was not raised: %+v", u)
	}
	if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") {
		t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
	}
	// And the report says so, with what was found — port, range, peers — and never the key.
	if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 ||
		report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
		t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
	}
	for _, o := range report.Outcomes {
		if strings.Contains(o.Detail, foundKey) {
			t.Errorf("the found key was printed in an outcome: %+v", o)
		}
	}
	if strings.Contains(report.Tunnel.Note, foundKey) {
		t.Error("the found key was printed in the account")
	}
	if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
		!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
		t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
	}
	if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded {
		t.Error("the found configuration was recorded as applied, so it would be removed as an orphan")
	}
}

func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	otherKey := filepath.Join(dir, "other.key")
	k, _ := ecdh.X25519().GenerateKey(rand.Reader)
	if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	cases := map[string]*declaration.Declaration{
		"another port":    aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"),
		"another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"),
		"another key":     aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"),
		"no key file":     aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"),
	}
	for name, d := range cases {
		m.asked = nil
		report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{},
			store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
		if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") {
			t.Fatalf("%s: the takeover was not refused: %v", name, err)
		}
		if name == "another key" && !strings.Contains(err.Error(), "overlay take") {
			t.Errorf("%s: the refusal does not name the remedy: %v", name, err)
		}
		if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") {
			t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name)
		}
		if m.units["wg-quick@mesh0"].active == "active" {
			t.Fatalf("%s: the mesh's interface was started on top of the found one", name)
		}
		if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") {
			t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel)
		}
		if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
			t.Errorf("%s: the found configuration was not kept before the refusal", name)
		}
	}
}

func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.units["wg-quick@mesh0"].wontStart = true
	report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
		store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
	if err == nil {
		t.Fatal("a mesh interface that did not come up was reported as applied")
	}
	if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") {
		t.Fatalf("the found unit was not stopped and then started again: %v", m.asked)
	}
	if m.units["wg-quick@wg0"].active != "active" {
		t.Fatal("the machine was left with no tunnel at all")
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken ||
		!strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") {
		t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel)
	}
}

func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	m.asked = nil

	report, again := applyAdopted(t, d, state, m, dir)
	if report.Changed() {
		t.Errorf("a second apply moved the machine: %+v", report.Outcomes)
	}
	if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still {
		t.Error("the hold on the found configuration was forgotten while the service still declares it")
	}
	if m.did("systemctl stop wg-quick@wg0") {
		t.Error("a found unit already down was stopped again")
	}
	if report.Tunnel == nil || report.Tunnel.State != Taken {
		t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel)
	}

	// Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh —
	// on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said.
	m.units["wg-quick@wg0"].active = "active"
	m.asked = nil
	report, _ = applyAdopted(t, d, again, m, dir)
	if m.did("systemctl stop wg-quick@wg0") {
		t.Error("a found unit started again by hand was stopped by the mesh")
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") {
		t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel)
	}
}

func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	// The unit is not running, yet the interface is up: the predecessor raised it by hand.
	m.units["wg-quick@wg0"].active = "inactive"
	m.wgUp = "wg0 mesh0\n"
	report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
		store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
	if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") {
		t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err)
	}
	if m.units["wg-quick@mesh0"].active == "active" {
		t.Error("the mesh's interface was started on a port the found one still holds")
	}
	// Looked at more than once before giving up: a person taking it down takes a moment.
	shows := 0
	for _, a := range m.asked {
		if a == "wg show interfaces" {
			shows++
		}
	}
	if shows < takeoverRechecks+1 {
		t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows)
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken {
		t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel)
	}
	if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
		t.Error("the found configuration was not kept before the refusal")
	}
}

func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) {
	_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
	  {"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
	   "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`))
	if err == nil || !strings.Contains(err.Error(), "adopted") {
		t.Fatalf("a takeover on a converged node was accepted: %v", err)
	}
}