mesh-host/internal/apply/takeover_test.go

package apply

import (
	"crypto/ecdh"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"git.novox.be/novox/mesh-sdk/go/declaration"
	"github.com/novox/mesh-host/internal/store"
)

// novox/hq ADR 0105: the host raises the mesh's interface with the found key and peers, stops the
// found interface without flushing it, and keeps its configuration — and stops nothing until the
// mesh's interface is known to be able to replace it.

// foundKey is the predecessor's private key, a real one made once per run: the key is what the
// takeover must never print or copy, so it had better be one.
var foundKey = func() string {
	k, err := ecdh.X25519().GenerateKey(rand.Reader)
	if err != nil {
		panic(err)
	}
	return base64.StdEncoding.EncodeToString(k.Bytes())
}()

var foundConf = "[Interface]\nPrivateKey = " + foundKey + "\n" +
	"ListenPort = 51900\nAddress = 192.0.2.1/24\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n" +
	"\n[Peer]\nPublicKey = PEER-B=\nAllowedIPs = 192.0.2.3/32\n"

// aTakeover is the private network's declaration for an adopted hub whose interface takes over
// the found tunnel: the mesh's configuration — on the found port and address, its key set from the
// node's own key file, the found peers in its list — and the interface's service naming what it
// replaces. Port and address are parameters so a test can declare a wrong one.
func aTakeover(t *testing.T, config, mesh, keyFile, port, address string) *declaration.Declaration {
	t.Helper()
	return adopted(t,
		`{"taken":[],"untaken":{"mesh-wireguard":["mesh-wireguard.overlay-config","mesh-wireguard.overlay-up"]}}`,
		`{"id":"mesh-wireguard.overlay-config","type":"file","path":"`+mesh+`","mode":"0600",
		  "content":"[Interface]\nAddress = `+address+`/32\nListenPort = `+port+`\nPostUp = wg set %i private-key `+keyFile+`\n\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"},
		 {"id":"mesh-wireguard.overlay-up","type":"service","unit":"wg-quick@mesh0","state":"running","boot":"enabled",
		  "restart-on":["mesh-wireguard.overlay-config"],
		  "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"`+config+`"}}`)
}

// aHubInUse is a machine with the predecessor's tunnel up and the mesh's not yet: the found
// configuration on disk, and the node's key file holding the found key, as enrolment left it.
func aHubInUse(t *testing.T) (dir, config, mesh, keyFile string, m *machine) {
	t.Helper()
	dir = t.TempDir()
	config = filepath.Join(dir, "wg0.conf")
	mesh = filepath.Join(dir, "mesh0.conf")
	keyFile = filepath.Join(dir, "overlay.key")
	if err := os.WriteFile(config, []byte(foundConf), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(keyFile, []byte(foundKey+"\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	m = &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{
		"wg-quick@wg0":   {active: "active", enabled: "enabled"},
		"wg-quick@mesh0": {active: "inactive", enabled: "disabled", fragment: "/usr/lib/systemd/system/wg-quick@.service"},
	}}
	takeoverRecheck = 0
	// The found configuration lives in this test's own wireguard directory (novox/hq ADR 0119).
	was := wireguardDir
	wireguardDir = dir
	t.Cleanup(func() { wireguardDir = was })
	return dir, config, mesh, keyFile, m
}

func TestTheFoundTunnelIsStoppedNeverFlushedAndItsConfigurationKept(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)

	// The found interface: its unit stopped and disabled, and nothing else done to it.
	if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Fatalf("the found unit was not stopped and disabled: %+v", u)
	}
	for _, asked := range m.asked {
		// Only ever asked about: which interfaces are up, and whether a peer has handshaken with
		// the mesh's own (novox/hq ADR 0119).
		if strings.HasPrefix(asked, "wg ") && !strings.HasPrefix(asked, "wg show interfaces") &&
			asked != "wg show mesh0 latest-handshakes" {
			t.Errorf("the found interface was touched with %q; it is stopped, never flushed", asked)
		}
		if strings.HasPrefix(asked, "wg-quick") || strings.Contains(asked, "peer remove") {
			t.Errorf("the found interface was flushed: %q", asked)
		}
	}
	// Its configuration: on disk as it was, its original kept, held for the module.
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatalf("the found configuration was changed:\n%s", got)
	}
	held, ok := state.HeldAt("mesh-wireguard.overlay-up.takes-over")
	if !ok || held.Kind != "file" || held.Target != config || held.Kept == "" || held.Module != "mesh-wireguard" {
		t.Fatalf("the found configuration is not held: %+v", held)
	}
	if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
		t.Fatalf("the original was not kept as found: %q", kept)
	}
	// The mesh's interface: up, enabled, with the found peers in the file the mesh wrote.
	if u := m.units["wg-quick@mesh0"]; u.active != "active" || u.enabled != "enabled" {
		t.Fatalf("the mesh's interface was not raised: %+v", u)
	}
	if got, _ := os.ReadFile(mesh); !strings.Contains(string(got), "PEER-A=") || strings.Contains(string(got), "PrivateKey") {
		t.Fatalf("the mesh's configuration does not carry the found peer, or carries a key:\n%s", got)
	}
	// And the report says so, with what was found — port, range, peers — and never the key.
	if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Port != 51900 ||
		report.Tunnel.Range != "192.0.2.0/24" || report.Tunnel.Peers != 2 || report.Tunnel.Kept != held.Kept {
		t.Fatalf("the report does not say what was carried: %+v", report.Tunnel)
	}
	for _, o := range report.Outcomes {
		if strings.Contains(o.Detail, foundKey) {
			t.Errorf("the found key was printed in an outcome: %+v", o)
		}
	}
	if strings.Contains(report.Tunnel.Note, foundKey) {
		t.Error("the found key was printed in the account")
	}
	if o := outcomeOf(report, "mesh-wireguard.overlay-up.takes-over"); o.Action != "held" ||
		!strings.Contains(o.Detail, "stopped wg-quick@wg0") || !strings.Contains(o.Detail, "never flushed") {
		t.Errorf("the takeover was not reported as a hold that stopped the found unit: %+v", o)
	}
	if _, recorded := state.Find("mesh-wireguard.overlay-up.takes-over"); recorded {
		t.Error("the found configuration was recorded as applied, so it would be removed as an orphan")
	}
}

func TestNothingIsStoppedUntilTheMeshsInterfaceCanReplaceTheFoundOne(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	otherKey := filepath.Join(dir, "other.key")
	k, _ := ecdh.X25519().GenerateKey(rand.Reader)
	if err := os.WriteFile(otherKey, []byte(base64.StdEncoding.EncodeToString(k.Bytes())+"\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	cases := map[string]*declaration.Declaration{
		"another port":    aTakeover(t, config, mesh, keyFile, "51821", "192.0.2.1"),
		"another address": aTakeover(t, config, mesh, keyFile, "51900", "10.42.0.1"),
		"another key":     aTakeover(t, config, mesh, otherKey, "51900", "192.0.2.1"),
		"no key file":     aTakeover(t, config, mesh, filepath.Join(dir, "missing.key"), "51900", "192.0.2.1"),
	}
	for name, d := range cases {
		m.asked = nil
		report, state, err := ApplyKeeping(t.Context(), archHost(t), d, store.State{},
			store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
		if err == nil || !strings.Contains(err.Error(), "would not replace the tunnel") {
			t.Fatalf("%s: the takeover was not refused: %v", name, err)
		}
		if name == "another key" && !strings.Contains(err.Error(), "overlay take") {
			t.Errorf("%s: the refusal does not name the remedy: %v", name, err)
		}
		if m.units["wg-quick@wg0"].active != "active" || m.did("systemctl stop wg-quick@wg0") {
			t.Fatalf("%s: the found unit was stopped although the mesh's interface could not replace it", name)
		}
		if m.units["wg-quick@mesh0"].active == "active" {
			t.Fatalf("%s: the mesh's interface was started on top of the found one", name)
		}
		if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "would not replace") {
			t.Fatalf("%s: the account does not say the tunnel is not taken and why: %+v", name, report.Tunnel)
		}
		if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
			t.Errorf("%s: the found configuration was not kept before the refusal", name)
		}
	}
}

func TestAMeshInterfaceThatFailsToStartGivesTheFoundOneBack(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.units["wg-quick@mesh0"].wontStart = true
	report, _, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
		store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
	if err == nil {
		t.Fatal("a mesh interface that did not come up was reported as applied")
	}
	if !m.did("systemctl stop wg-quick@wg0") || !m.did("systemctl start wg-quick@wg0") {
		t.Fatalf("the found unit was not stopped and then started again: %v", m.asked)
	}
	if m.units["wg-quick@wg0"].active != "active" {
		t.Fatal("the machine was left with no tunnel at all")
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken ||
		!strings.Contains(report.Tunnel.Note, "did not come up") || !strings.Contains(report.Tunnel.Note, "started again") {
		t.Fatalf("the account does not say the mesh's interface failed and the found one was given back: %+v", report.Tunnel)
	}
}

func TestATakeoverIsSteadyAndAFoundUnitUpAgainIsSaidNotStopped(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	m.asked = nil

	report, again := applyAdopted(t, d, state, m, dir)
	if report.Changed() {
		t.Errorf("a second apply moved the machine: %+v", report.Outcomes)
	}
	if _, still := again.HeldAt("mesh-wireguard.overlay-up.takes-over"); !still {
		t.Error("the hold on the found configuration was forgotten while the service still declares it")
	}
	if m.did("systemctl stop wg-quick@wg0") {
		t.Error("a found unit already down was stopped again")
	}
	if report.Tunnel == nil || report.Tunnel.State != Taken {
		t.Errorf("a steady takeover does not read as taken: %+v", report.Tunnel)
	}

	// Somebody starts the found unit again beside the mesh's interface. Not stopped by the mesh —
	// on the hub it cannot hold the port, on a spoke stopping it would be a fight — but said.
	m.units["wg-quick@wg0"].active = "active"
	m.asked = nil
	report, _ = applyAdopted(t, d, again, m, dir)
	if m.did("systemctl stop wg-quick@wg0") {
		t.Error("a found unit started again by hand was stopped by the mesh")
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken || !strings.Contains(report.Tunnel.Note, "running again beside") {
		t.Errorf("the account does not say the found unit is up again: %+v", report.Tunnel)
	}
}

func TestAFoundInterfaceRaisedByHandIsRefusedNamingTheRemedy(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	// The unit is not running, yet the interface is up: the predecessor raised it by hand.
	m.units["wg-quick@wg0"].active = "inactive"
	m.wgUp = "wg0 mesh0\n"
	report, state, err := ApplyKeeping(t.Context(), archHost(t), aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"),
		store.State{}, store.OriginDeclared, m.run, nil, nil, KeepIn(dir))
	if err == nil || !strings.Contains(err.Error(), "wg-quick down wg0") || !strings.Contains(err.Error(), "Nothing was flushed") {
		t.Fatalf("an interface raised by hand was not refused naming the remedy: %v", err)
	}
	if m.units["wg-quick@mesh0"].active == "active" {
		t.Error("the mesh's interface was started on a port the found one still holds")
	}
	// Looked at more than once before giving up: a person taking it down takes a moment.
	shows := 0
	for _, a := range m.asked {
		if a == "wg show interfaces" {
			shows++
		}
	}
	if shows < takeoverRechecks+1 {
		t.Errorf("the interface was looked at %d time(s) before the refusal; a person needs a moment", shows)
	}
	if report.Tunnel == nil || report.Tunnel.State != NotTaken {
		t.Errorf("the account does not say the tunnel is not taken: %+v", report.Tunnel)
	}
	if _, held := state.HeldAt("mesh-wireguard.overlay-up.takes-over"); !held {
		t.Error("the found configuration was not kept before the refusal")
	}
}

func TestATakeoverIsRefusedOnAConvergedDeclaration(t *testing.T) {
	_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[
	  {"id":"up","type":"service","unit":"wg-quick@mesh0","state":"running",
	   "takes-over":{"interface":"wg0","unit":"wg-quick@wg0","config":"/etc/wireguard/wg0.conf"}}]}`))
	if err == nil || !strings.Contains(err.Error(), "adopted") {
		t.Fatalf("a takeover on a converged node was accepted: %v", err)
	}
}

// novox/hq ADR 0119: once the take is proven — taken, and a peer handshaken on the mesh's
// interface — the found configuration is removed from where its unit reads it, its original stays
// kept and the hold on it ends. Never before, and never brought back.

const takesOverID = "mesh-wireguard.overlay-up.takes-over"

// handshaken is `wg show mesh0 latest-handshakes` with one of the two peers through.
const handshaken = "PEER-A=\t1790000000\nPEER-B=\t0\n"

func TestAProvenTakeRetiresTheFoundConfiguration(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.handshakes = handshaken
	report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)

	if _, err := os.Lstat(config); !os.IsNotExist(err) {
		t.Fatalf("a proven take left the found configuration where its unit reads it: %v", err)
	}
	retired, ok := state.RetiredAt(config)
	if !ok || retired.Kept == "" || retired.ID != takesOverID {
		t.Fatalf("the retirement was not recorded: %+v", state.Retired)
	}
	if kept, _ := os.ReadFile(retired.Kept); string(kept) != foundConf {
		t.Fatalf("the kept original did not survive the retirement: %q", kept)
	}
	if _, held := state.HeldAt(takesOverID); held {
		t.Error("the hold on the found configuration did not end with its retirement")
	}
	if u := m.units["wg-quick@wg0"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Errorf("the found unit is not left down and disabled: %+v", u)
	}
	if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
		!strings.Contains(report.Tunnel.Note, "proven: 1 peer(s) handshaken on mesh0") ||
		!strings.Contains(report.Tunnel.Note, "is retired") {
		t.Fatalf("the account does not say the take is proven and the configuration retired: %+v", report.Tunnel)
	}
	if o := outcomeOf(report, takesOverID); o.Action != "removed" || !strings.Contains(o.Detail, "retired") {
		t.Errorf("the retirement is not what the apply says it did to the file: %+v", o)
	}
	// And the account still carries what was found, read from the kept original.
	if report.Tunnel.Port != 51900 || report.Tunnel.Peers != 2 {
		t.Errorf("the account lost what the tunnel was: %+v", report.Tunnel)
	}
}

func TestATakeNotProvenKeepsTheFoundConfigurationAndSaysSo(t *testing.T) {
	cases := map[string]struct {
		handshakes string
		fail       error
		says       string
	}{
		"no peer at all":          {"", nil, "no peer has handshaken on mesh0"},
		"every handshake at zero": {"PEER-A=\t0\nPEER-B=\t0\n", nil, "no peer has handshaken on mesh0"},
		"wg is not there":         {"", errors.New(`exec: "wg": executable file not found in $PATH`), "executable file not found"},
		"the answer is nonsense":  {"unable to access interface\n", nil, "not a peer and a time"},
	}
	for name, c := range cases {
		dir, config, mesh, keyFile, m := aHubInUse(t)
		m.handshakes, m.handshakesFail = c.handshakes, c.fail
		d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
		report, state := applyAdopted(t, d, store.State{}, m, dir)

		if got, _ := os.ReadFile(config); string(got) != foundConf {
			t.Fatalf("%s: a take not proven lost the found configuration", name)
		}
		if _, held := state.HeldAt(takesOverID); !held {
			t.Errorf("%s: the hold ended although the take is not proven", name)
		}
		if _, retired := state.RetiredAt(config); retired {
			t.Errorf("%s: recorded as retired", name)
		}
		if report.Tunnel == nil || report.Tunnel.State != Taken ||
			!strings.Contains(report.Tunnel.Note, "taken, not yet proven") ||
			!strings.Contains(report.Tunnel.Note, c.says) || !strings.Contains(report.Tunnel.Note, "is kept") {
			t.Errorf("%s: the account does not say the take is not proven and why: %+v", name, report.Tunnel)
		}

		// A later apply that finds a peer through retires it: the take itself need not be the one.
		m.handshakes, m.handshakesFail = handshaken, nil
		_, state = applyAdopted(t, d, state, m, dir)
		if _, err := os.Lstat(config); !os.IsNotExist(err) {
			t.Errorf("%s: the apply after the take was proven kept the found configuration", name)
		}
		if _, retired := state.RetiredAt(config); !retired {
			t.Errorf("%s: the later retirement was not recorded", name)
		}
	}
}

func TestAFoundConfigurationWhoseKeptOriginalIsMissingIsNotRetired(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	held, _ := state.HeldAt(takesOverID)
	if err := os.Remove(held.Kept); err != nil {
		t.Fatal(err)
	}

	m.handshakes = handshaken
	report, state := applyAdopted(t, d, state, m, dir)
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatal("the found configuration was removed with no kept original left of it")
	}
	if _, still := state.HeldAt(takesOverID); !still {
		t.Error("the hold ended although nothing was retired")
	}
	if _, retired := state.RetiredAt(config); retired {
		t.Error("recorded as retired")
	}
	if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "is not retired") ||
		!strings.Contains(report.Tunnel.Note, held.Kept+" is missing") {
		t.Errorf("the account does not say the kept original is missing: %+v", report.Tunnel)
	}
}

func TestAFoundConfigurationRewrittenSinceItWasFoundIsKeptAgainBeforeItGoes(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	rewritten := foundConf + "\n[Peer]\nPublicKey = PEER-C=\nAllowedIPs = 192.0.2.4/32\n"
	if err := os.WriteFile(config, []byte(rewritten), 0o600); err != nil {
		t.Fatal(err)
	}

	m.handshakes = handshaken
	_, state = applyAdopted(t, d, state, m, dir)
	if _, err := os.Lstat(config); !os.IsNotExist(err) {
		t.Fatal("a proven take kept a rewritten configuration")
	}
	retired, _ := state.RetiredAt(config)
	if first, _ := os.ReadFile(retired.Kept); string(first) != foundConf {
		t.Errorf("the first original was overwritten: %q", first)
	}
	kept, _ := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
	var found bool
	for _, k := range kept {
		if got, _ := os.ReadFile(k); string(got) == rewritten {
			found = true
		}
	}
	if !found {
		t.Errorf("what the file held when it was retired was not kept: %v", kept)
	}
}

func TestARetiredTakeIsSteadyAndItsFoundUnitFindsNothingToDo(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.handshakes = handshaken
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	retired, _ := state.RetiredAt(config)

	// wg-quick@wg0 with no configuration: inactive, and disabled — the check of it every apply
	// makes must find nothing to do and fail on nothing.
	m.asked = nil
	report, again := applyAdopted(t, d, state, m, dir)
	if report.Changed() {
		t.Errorf("an apply after the retirement moved the machine: %+v", report.Outcomes)
	}
	if m.did("systemctl stop wg-quick@wg0") || m.did("systemctl start wg-quick@wg0") {
		t.Errorf("the retired tunnel's unit was acted on: %v", m.asked)
	}
	if _, err := os.Lstat(config); !os.IsNotExist(err) {
		t.Error("the found configuration came back")
	}
	if _, held := again.HeldAt(takesOverID); held {
		t.Error("a retired configuration is held again")
	}
	if r, ok := again.RetiredAt(config); !ok || r != retired {
		t.Errorf("the retirement was not kept as it was: %+v", again.Retired)
	}
	if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "retired") {
		t.Errorf("the retired configuration is not said as retired: %+v", o)
	}
	if report.Tunnel == nil || report.Tunnel.State != Taken || report.Tunnel.Kept != retired.Kept ||
		!strings.Contains(report.Tunnel.Note, "retired") || report.Tunnel.Port != 51900 {
		t.Errorf("the account of a retired take does not say so: %+v", report.Tunnel)
	}

	// Enabled at boot again by a person: disabled again, as any take does, and still no error.
	m.units["wg-quick@wg0"].enabled = "enabled"
	_, _ = applyAdopted(t, d, again, m, dir)
	if m.units["wg-quick@wg0"].enabled != "disabled" {
		t.Error("the found unit enabled again by hand was left to start at boot")
	}
}

func TestUndeclaringThePrivateNetworkAfterRetirementBringsNothingBack(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.handshakes = handshaken
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)

	// The private network unassigned: only something else is declared.
	other := adopted(t, `{"taken":[],"untaken":{}}`,
		`{"id":"other.file","type":"file","path":"`+filepath.Join(dir, "other.conf")+`","content":"x\n"}`)
	m.asked = nil
	_, after := applyAdopted(t, other, state, m, dir)
	if _, err := os.Lstat(config); !os.IsNotExist(err) {
		t.Fatal("undeclaring the private network brought the found configuration back")
	}
	if m.did("systemctl start wg-quick@wg0") || m.did("systemctl enable wg-quick@wg0") {
		t.Errorf("undeclaring the private network started the found tunnel: %v", m.asked)
	}
	if _, ok := after.RetiredAt(config); !ok {
		t.Error("the retirement was forgotten with the private network")
	}

	// Assigned again, it finds the configuration retired rather than missing, and raises the
	// mesh's interface.
	report, _ := applyAdopted(t, d, after, m, dir)
	if report.Tunnel == nil || report.Tunnel.State != Taken {
		t.Errorf("the private network assigned again did not take the tunnel: %+v", report.Tunnel)
	}
	if _, err := os.Lstat(config); !os.IsNotExist(err) {
		t.Error("assigning the private network again brought the found configuration back")
	}
}

func TestAPlanSaysTheFoundConfigurationIsRetiredWhenTheTakeIsProven(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")

	plan := Plan(d, store.State{}, store.OriginDeclared)
	report, state := applyAdopted(t, d, store.State{}, m, dir)
	if got, want := strings.Join(ids(plan), " "), strings.Join(outcomeIDs(report), " "); got != want {
		t.Errorf("the plan said %q and the apply did %q", got, want)
	}
	var take Step
	for _, s := range plan {
		if s.ID == takesOverID {
			take = s
		}
	}
	if take.Verb != "hold" || take.Target != config || !strings.Contains(take.Why, "retired — removed from "+config) ||
		!strings.Contains(take.Why, "handshaking on mesh0") {
		t.Errorf("the plan does not say the found configuration is retired once proven: %+v", take)
	}
	// Held and still declared: never planned as forgotten.
	if strings.Contains(verbs(Plan(d, state, store.OriginDeclared)), "forget "+takesOverID) {
		t.Error("the plan forgets a hold the apply keeps")
	}

	m.handshakes = handshaken
	_, state = applyAdopted(t, d, state, m, dir)
	for _, s := range Plan(d, state, store.OriginDeclared) {
		if s.ID == takesOverID && (s.Verb != "check" || !strings.Contains(s.Why, "retired once the take")) {
			t.Errorf("a retired configuration is planned as %+v", s)
		}
	}
}

// keptCopies is every copy kept of the found configuration.
func keptCopies(t *testing.T, dir string) []string {
	t.Helper()
	kept, err := filepath.Glob(filepath.Join(dir, "kept", "*-wg0.conf"))
	if err != nil {
		t.Fatal(err)
	}
	return kept
}

func TestAConfigurationPutBackIsRetiredAgainOnItsFirstOriginalAndSettles(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	m.handshakes = handshaken
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	first, _ := state.RetiredAt(config)

	// Put back by hand with the original, while no peer is through yet: held on the first
	// original, and the account says what a rollback takes.
	write(t, config, foundConf)
	m.handshakes = ""
	report, state := applyAdopted(t, d, state, m, dir)
	if held, ok := state.HeldAt(takesOverID); !ok || held.Kept != first.Kept {
		t.Fatalf("what came back is not held on the first original: %+v", held)
	}
	if report.Tunnel == nil || !strings.Contains(report.Tunnel.Note, "came back after it was retired") ||
		!strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
		t.Errorf("the account does not say a rollback means unassigning the private network: %+v", report.Tunnel)
	}

	// Proven: retired again, nothing more kept, said once.
	m.handshakes = handshaken
	report, state = applyAdopted(t, d, state, m, dir)
	again, _ := state.RetiredAt(config)
	if _, err := os.Lstat(config); !os.IsNotExist(err) || again.Kept != first.Kept || again.Extra != "" {
		t.Fatalf("put back as it was, it was not retired again on the first original: %+v", again)
	}
	if o := outcomeOf(report, takesOverID); o.Action != "removed" ||
		!strings.HasPrefix(o.Detail, "the found configuration came back and was retired again") ||
		strings.Contains(o.Detail, "differed") {
		t.Errorf("the second retirement is not said as one: %+v", o)
	}
	if !strings.Contains(report.Tunnel.Note, "unassigning the private network first") {
		t.Errorf("the account does not say what a rollback takes: %q", report.Tunnel.Note)
	}
	if n := len(keptCopies(t, dir)); n != 1 {
		t.Errorf("%d copies kept of one content", n)
	}
	if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
		t.Errorf("a steady machine moved after the second retirement: %+v", report.Outcomes)
	}

	// Put back with something else: that is kept beside the first original, which stays the record's.
	other := strings.Replace(foundConf, "PEER-B=", "PEER-Z=", 1)
	write(t, config, other)
	report, state = applyAdopted(t, d, state, m, dir)
	third, _ := state.RetiredAt(config)
	if third.Kept != first.Kept || third.Extra == "" || third.Extra == first.Kept {
		t.Fatalf("other content was not kept apart from the first original: %+v", third)
	}
	if got, _ := os.ReadFile(third.Extra); string(got) != other {
		t.Errorf("the extra copy does not hold what was put back: %q", got)
	}
	if o := outcomeOf(report, takesOverID); !strings.Contains(o.Detail, third.Extra) ||
		!strings.Contains(report.Tunnel.Note, third.Extra) {
		t.Errorf("where the extra copy is was not said: %+v / %q", o, report.Tunnel.Note)
	}

	// And the same other content again: nothing more kept, the record as it was.
	write(t, config, other)
	report, state = applyAdopted(t, d, state, m, dir)
	fourth, _ := state.RetiredAt(config)
	if fourth.Kept != first.Kept || fourth.Extra != third.Extra || len(keptCopies(t, dir)) != 2 {
		t.Errorf("the same content put back again grew the copies: %+v, %v", fourth, keptCopies(t, dir))
	}
	if o := outcomeOf(report, takesOverID); strings.Contains(o.Detail, "differed") {
		t.Errorf("a copy already kept was said as new: %+v", o)
	}
	if report, _ := applyAdopted(t, d, state, m, dir); report.Changed() {
		t.Errorf("a steady machine moved: %+v", report.Outcomes)
	}
}

func TestOnlyWgQuicksOwnConfigurationIsRetired(t *testing.T) {
	// Not under the wireguard directory.
	dir, config, mesh, keyFile, m := aHubInUse(t)
	wireguardDir = filepath.Join(dir, "elsewhere")
	m.handshakes = handshaken
	report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatal("a configuration outside wg-quick's directory was removed")
	}
	if _, held := state.HeldAt(takesOverID); !held || !strings.Contains(report.Tunnel.Note, "is not retired: only ") {
		t.Errorf("the refusal is not said, or the hold ended: %+v", report.Tunnel)
	}

	// A path the mesh itself writes.
	dir, config, mesh, keyFile, m = aHubInUse(t)
	m.handshakes = handshaken
	known := store.State{}
	known.Record(store.Applied{ID: "bundle.wg0", Type: "file", Target: config, Origin: store.OriginCarried})
	report, _ = applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), known, m, dir)
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatal("a path the mesh writes was retired")
	}
	if !strings.Contains(report.Tunnel.Note, "a path the mesh itself writes") {
		t.Errorf("the refusal is not said: %+v", report.Tunnel)
	}
}

func TestAFoundConfigurationThatIsALinkIsKeptAndLeftToAPerson(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	target := filepath.Join(dir, "predecessor", "hub.conf")
	if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
		t.Fatal(err)
	}
	write(t, target, foundConf)
	if err := os.Remove(config); err != nil {
		t.Fatal(err)
	}
	if err := os.Symlink(target, config); err != nil {
		t.Fatal(err)
	}
	m.handshakes = handshaken
	report, state := applyAdopted(t, aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1"), store.State{}, m, dir)
	if _, err := os.Lstat(config); err != nil {
		t.Fatal("the link was removed, leaving the key-bearing file it points at")
	}
	if got, _ := os.ReadFile(target); string(got) != foundConf {
		t.Fatal("the file the link points at was touched")
	}
	held, ok := state.HeldAt(takesOverID)
	if !ok {
		t.Fatal("the hold ended")
	}
	if kept, _ := os.ReadFile(held.Kept); string(kept) != foundConf {
		t.Errorf("what was kept is not what the link points at: %q", kept)
	}
	if !strings.Contains(report.Tunnel.Note, "is a link to "+target) || !strings.Contains(report.Tunnel.Note, "by hand") {
		t.Errorf("the account does not say the link must be retired by hand: %q", report.Tunnel.Note)
	}
}

func TestARetirementWhoseRecordWasNeverSavedIsRecordedByTheNextApply(t *testing.T) {
	dir, config, mesh, keyFile, m := aHubInUse(t)
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)
	held, _ := state.HeldAt(takesOverID)
	// An apply removed the file and stopped before its state was saved.
	if err := os.Remove(config); err != nil {
		t.Fatal(err)
	}
	m.handshakes = handshaken
	report, state := applyAdopted(t, d, state, m, dir)
	if r, ok := state.RetiredAt(config); !ok || r.Kept != held.Kept {
		t.Fatalf("the retirement was not recorded: %+v", state.Retired)
	}
	if _, still := state.HeldAt(takesOverID); still {
		t.Error("the hold did not end")
	}
	if o := outcomeOf(report, takesOverID); o.Action != "unchanged" || !strings.Contains(o.Detail, "already gone") {
		t.Errorf("a file already gone is not said as such: %+v", o)
	}
}

func TestARemovalThatFailsKeepsTheFileAndTheHold(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root removes from a directory it may not write to")
	}
	dir, _, mesh, keyFile, m := aHubInUse(t)
	wg := filepath.Join(dir, "wireguard")
	if err := os.MkdirAll(wg, 0o700); err != nil {
		t.Fatal(err)
	}
	config := filepath.Join(wg, "wg0.conf")
	write(t, config, foundConf)
	wireguardDir = wg
	d := aTakeover(t, config, mesh, keyFile, "51900", "192.0.2.1")
	_, state := applyAdopted(t, d, store.State{}, m, dir)

	if err := os.Chmod(wg, 0o500); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chmod(wg, 0o700) })
	m.handshakes = handshaken
	report, state := applyAdopted(t, d, state, m, dir)
	if got, _ := os.ReadFile(config); string(got) != foundConf {
		t.Fatal("the found configuration is gone although it could not be removed")
	}
	if _, held := state.HeldAt(takesOverID); !held {
		t.Error("the hold ended although nothing was retired")
	}
	if _, retired := state.RetiredAt(config); retired {
		t.Error("recorded as retired")
	}
	if !strings.Contains(report.Tunnel.Note, "removing it failed") || !strings.Contains(report.Tunnel.Note, "permission denied") {
		t.Errorf("the failed removal is not said: %q", report.Tunnel.Note)
	}
}