mesh-host/internal/apply/found_test.go

package apply

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/novox/mesh-host/internal/store"
)

// Defends novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed,
// and leaves what was the machine's — which needs what was found kept exactly, and a unit the mesh
// made known for the mesh's whatever its record says.

func unitsMachine(units map[string]*fakeUnit) *machine {
	return &machine{containers: map[string]*fakeContainer{}, units: units}
}

// nothingButA is a declaration of one unrelated file, so everything recorded is undeclared.
func nothingButA(t *testing.T) string {
	return `{"declaration":1,"resources":[
	  {"id":"other","type":"file","path":"` + filepath.Join(t.TempDir(), "a") + `","content":"a\n"}]}`
}

// copyOf is a state as a later load of it would be: sharing nothing with the one it came from.
func copyOf(t *testing.T, s store.State) store.State {
	t.Helper()
	raw, err := json.Marshal(s)
	if err != nil {
		t.Fatal(err)
	}
	var out store.State
	if err := json.Unmarshal(raw, &out); err != nil {
		t.Fatal(err)
	}
	return out
}

func applyOn(t *testing.T, raw string, known store.State, m *machine) (Report, store.State, error) {
	t.Helper()
	return Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, m.run, nil, nil)
}

func TestAUnitWhoseFileTheMeshWroteIsStoppedWhateverItsRecordSays(t *testing.T) {
	// The adoption guard's unit file is the mesh's own file resource, written where there was none.
	// Its service recorded before the host kept what it found has no Found, and forgetting it left
	// the guard's table loaded on a converged node and its unit file deleted from under it.
	units := t.TempDir()
	was := unitDir
	unitDir = units
	t.Cleanup(func() { unitDir = was })
	unitFile := filepath.Join(units, "mesh-guard.service")
	if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	m := unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
	fileThereAtStop := false
	run := func(ctx context.Context, name string, args ...string) (string, error) {
		if name == "systemctl" && args[0] == "stop" {
			_, err := os.Stat(unitFile)
			fileThereAtStop = err == nil
		}
		return m.run(ctx, name, args...)
	}
	// As a host before this change recorded them: the unit file first, then the service it
	// starts, and no Found on the service.
	known := store.State{Resources: []store.Applied{
		{ID: "adoption.guard-unit", Type: "file", Target: unitFile, Origin: store.OriginDeclared},
		{ID: "adoption.guard-running", Type: "service", Target: "mesh-guard.service", Origin: store.OriginDeclared},
	}}
	report, after, err := applyWith(t, parse(t, nothingButA(t)), known, run)
	if err != nil {
		t.Fatal(err)
	}
	if u := m.units["mesh-guard.service"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Errorf("the mesh's own unit was left %s and %s: %v", u.active, u.enabled, m.asked)
	}
	if !fileThereAtStop {
		t.Error("the unit was stopped after its file was deleted, or not at all")
	}
	if o := outcomeOf(report, "adoption.guard-running"); o.Action != "removed" || !strings.Contains(o.Detail, "unit file") {
		t.Errorf("outcome %+v", o)
	}
	if _, err := os.Stat(unitFile); !os.IsNotExist(err) {
		t.Error("the unit file outlived its record")
	}
	if len(after.Resources) != 1 {
		t.Errorf("still recorded: %v", after.IDs())
	}

	// A unit file the host wrote OVER — its original kept — is the machine's unit, and a record
	// with no Found leaves it as it is.
	if err := os.WriteFile(unitFile, []byte("[Unit]\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	m = unitsMachine(map[string]*fakeUnit{"mesh-guard.service": {active: "active", enabled: "enabled"}})
	known.Resources[0].Kept = filepath.Join(t.TempDir(), "original")
	if err := os.WriteFile(known.Resources[0].Kept, []byte("[Unit]\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	if _, _, err := applyWith(t, parse(t, nothingButA(t)), known, m.run); err != nil {
		t.Fatal(err)
	}
	if m.did("systemctl stop") || m.did("systemctl disable") {
		t.Errorf("a unit whose file the mesh only wrote over was stopped: %v", m.asked)
	}
}

func TestOnlyAUnitsOwnFileTheMeshCreatedMakesItTheMeshs(t *testing.T) {
	known := store.State{Resources: []store.Applied{
		{ID: "a", Type: "file", Target: "/etc/systemd/system/made.service"},
		{ID: "b", Type: "file", Target: "/run/systemd/system/runtime.service"},
		{ID: "c", Type: "file", Target: "/etc/systemd/system/kept.service", Kept: "/var/lib/mesh-host/kept/x"},
		{ID: "d", Type: "file", Target: "/etc/systemd/system/into.service", Into: &store.Into{Format: "block"}},
		{ID: "e", Type: "file", Target: "/etc/systemd/system/docker.service.d/mesh.conf"},
		{ID: "f", Type: "file", Target: "/etc/mesh/elsewhere.service"},
		{ID: "g", Type: "directory", Target: "/etc/systemd/system/dir.service"},
	}}
	made := meshMadeUnits(known)
	for unit, want := range map[string]bool{"made.service": true, "runtime.service": true, "kept.service": false,
		"into.service": false, "mesh.conf": false, "docker.service.d": false, "elsewhere.service": false, "dir.service": false} {
		if made[unit] != want {
			t.Errorf("%s: made %v, want %v", unit, made[unit], want)
		}
	}
}

func TestWhatWasFoundOutlivesAFirstApplyThatFailed(t *testing.T) {
	// Enabled, then it would not start: no record. The next apply must not read the enable as
	// the machine's — or undeclaring leaves enabled a unit the mesh enabled.
	m := unitsMachine(map[string]*fakeUnit{"filter.service": {active: "inactive", enabled: "disabled", wontStart: true}})
	declared := `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"filter.service","state":"running","boot":"enabled"}]}`
	_, first, err := applyOn(t, declared, store.State{}, m)
	if err == nil || !m.did("systemctl enable filter.service") {
		t.Fatalf("the fixture did not enable and then fail: %v, %v", err, m.asked)
	}
	if _, ok := first.Find("s"); ok {
		t.Fatal("a failed apply was recorded")
	}
	if p := first.FoundFirst["s"]; p.State != "stopped" || p.Boot != "disabled" || p.Unit != "filter.service" {
		t.Fatalf("what was found was not kept through the failure: %+v", first.FoundFirst)
	}

	failed := copyOf(t, first)

	m.units["filter.service"].wontStart = false
	_, second, err := applyOn(t, declared, first, m)
	if err != nil {
		t.Fatal(err)
	}
	if rec, _ := second.Find("s"); rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "disabled" {
		t.Errorf("the record carries the mesh's own effect as found: %+v", rec.Found)
	}
	if second.FoundFirst != nil {
		t.Errorf("kept apart after the record carried it: %+v", second.FoundFirst)
	}

	if _, _, err := applyOn(t, nothingButA(t), second, m); err != nil {
		t.Fatal(err)
	}
	if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Errorf("undeclared, the unit was left %s and %s", u.active, u.enabled)
	}

	// Undeclared before it was ever recorded, what was found goes with it — only for its origin.
	if _, kept, _ := Apply(context.Background(), archHost(t), parse(t, nothingButA(t)), copyOf(t, failed),
		store.OriginCarried, m.run, nil, nil); kept.FoundFirst["s"].State == "" {
		t.Error("a carried apply dropped what the mesh's declaration found")
	}
	if _, dropped, err := applyOn(t, nothingButA(t), copyOf(t, failed), m); err != nil || dropped.FoundFirst != nil {
		t.Errorf("kept for a service no longer declared: %+v, %v", dropped.FoundFirst, err)
	}
}

func TestBootIsFoundTheFirstTimeTheMeshSetsIt(t *testing.T) {
	// The declaration said nothing about boot at first, so the mesh never touched it: what is
	// there when a declaration first does is still the machine's.
	m := unitsMachine(map[string]*fakeUnit{"web.service": {active: "active", enabled: "disabled"}})
	known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "web.service",
		Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "web.service", State: "running"}}}}
	_, after, err := applyOn(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"web.service","state":"running","boot":"enabled"}]}`, known, m)
	if err != nil {
		t.Fatal(err)
	}
	rec, _ := after.Find("s")
	if rec.Found == nil || rec.Found.State != "running" || rec.Found.Boot != "disabled" {
		t.Fatalf("found %+v, want running and disabled", rec.Found)
	}
	report, _, err := applyOn(t, nothingButA(t), after, m)
	if err != nil {
		t.Fatal(err)
	}
	if u := m.units["web.service"]; u.active != "active" || u.enabled != "disabled" {
		t.Errorf("undeclared, the unit is %s and %s; want running, and disabled again", u.active, u.enabled)
	}
	if o := outcomeOf(report, "s"); o.Action != "restored" || o.Detail != "disabled at boot, as the host found it" {
		t.Errorf("outcome %+v", o)
	}
}

func TestAServiceOnceDeclaredWithNoStateIsFoundWhenFirstGivenOne(t *testing.T) {
	// Declared with no state (novox/hq ADR 0117), the mesh never started or stopped it — so what
	// is there when a declaration first gives it one is what the machine had.
	m := unitsMachine(map[string]*fakeUnit{"net.service": {active: "inactive", enabled: "disabled"}})
	known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "net.service",
		Origin: store.OriginDeclared, Stateless: true}}}
	_, after, err := applyOn(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"net.service","state":"running"}]}`, known, m)
	if err != nil {
		t.Fatal(err)
	}
	if rec, _ := after.Find("s"); rec.Found == nil || rec.Found.State != "stopped" {
		t.Fatalf("found %+v, want stopped", rec.Found)
	}
	if _, _, err := applyOn(t, nothingButA(t), after, m); err != nil {
		t.Fatal(err)
	}
	if m.units["net.service"].active != "inactive" {
		t.Error("the unit the mesh started outlived its declaration")
	}
}

func TestAUnitWrittenInTheSameApplyIsLoadedBeforeItIsRead(t *testing.T) {
	// Read before the service manager is told about its new file, the unit is not there to find.
	dir := t.TempDir()
	m := unitsMachine(map[string]*fakeUnit{"fresh.service": {active: "inactive", enabled: "disabled"}})
	_, _, err := applyOn(t, `{"declaration":1,"resources":[
	  {"id":"unit","type":"file","path":"`+filepath.Join(dir, "fresh.service")+`","content":"[Unit]\n"},
	  {"id":"s","type":"service","unit":"fresh.service","state":"running","restart-on":["unit"]}]}`,
		store.State{}, m)
	if err != nil {
		t.Fatal(err)
	}
	reload, show := -1, -1
	for i, a := range m.asked {
		if a == "systemctl daemon-reload" && reload < 0 {
			reload = i
		}
		if strings.HasPrefix(a, "systemctl show fresh.service") && show < 0 {
			show = i
		}
	}
	if reload < 0 || show < 0 || reload > show {
		t.Errorf("the unit was read before its file was loaded: %v", m.asked)
	}
}

func TestAServiceMovedToAnotherUnitGivesTheOldOneBackAndFindsTheNewOne(t *testing.T) {
	m := unitsMachine(map[string]*fakeUnit{
		"old.service": {active: "active", enabled: "enabled"},
		"new.service": {active: "inactive", enabled: "disabled"},
	})
	known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "old.service",
		Origin: store.OriginDeclared, Found: &store.FoundUnit{Unit: "old.service", State: "stopped"}}}}
	report, after, err := applyOn(t, `{"declaration":1,"resources":[
	  {"id":"s","type":"service","unit":"new.service","state":"running"}]}`, known, m)
	if err != nil {
		t.Fatal(err)
	}
	if m.units["old.service"].active != "inactive" {
		t.Error("the unit the mesh started is still running though nothing declares it")
	}
	if m.units["new.service"].active != "active" {
		t.Error("the unit now declared was not started")
	}
	rec, _ := after.Find("s")
	if rec.Found == nil || rec.Found.Unit != "new.service" || rec.Found.State != "stopped" {
		t.Errorf("what was found about the old unit was carried to the new one: %+v", rec.Found)
	}
	if o := outcomeOf(report, "s"); !strings.Contains(o.Detail, "old.service stopped, as the host found it") {
		t.Errorf("giving the old unit back went unsaid: %+v", o)
	}
}

func TestARemovalSaysWhatItDid(t *testing.T) {
	cases := []struct {
		name           string
		found          *store.FoundUnit
		unit           *fakeUnit
		action, detail string
	}{
		{"found stopped and still stopped", &store.FoundUnit{State: "stopped"},
			&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten", "already as the host found it (stopped)"},
		{"started by the mesh", &store.FoundUnit{State: "stopped"},
			&fakeUnit{active: "active", enabled: "disabled"}, "restored", "stopped, as the host found it"},
		{"started and enabled by the mesh", &store.FoundUnit{State: "stopped", Boot: "disabled"},
			&fakeUnit{active: "active", enabled: "enabled"}, "restored", "stopped, disabled at boot, as the host found it"},
		{"found running, stopped by the mesh", &store.FoundUnit{State: "running"},
			&fakeUnit{active: "inactive", enabled: "disabled"}, "forgotten",
			"left as it is; the mesh stopped it and does not start anything on the way out"},
		{"found running and enabled, disabled by the mesh", &store.FoundUnit{State: "running", Boot: "enabled"},
			&fakeUnit{active: "active", enabled: "disabled"}, "forgotten",
			"left as it is; the mesh disabled it at boot and does not start anything on the way out"},
		{"found running, still running", &store.FoundUnit{State: "running"},
			&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "it was running before the mesh; left as it is"},
		// Found says to stop it, so the machine is asked about it — and it is gone.
		{"started by the mesh, since uninstalled", &store.FoundUnit{State: "stopped"},
			nil, "forgotten", "the unit no longer exists"},
		{"recorded before the host kept what it found", nil,
			&fakeUnit{active: "active", enabled: "enabled"}, "forgotten", "recorded before the host kept what it found; left as it is"},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			units := map[string]*fakeUnit{}
			if c.unit != nil {
				units["unit.service"] = c.unit
			}
			m := unitsMachine(units)
			known := store.State{Resources: []store.Applied{{ID: "s", Type: "service", Target: "unit.service",
				Origin: store.OriginDeclared, Found: c.found}}}
			report, after, err := applyOn(t, nothingButA(t), known, m)
			if err != nil {
				t.Fatal(err)
			}
			if o := outcomeOf(report, "s"); o.Action != c.action || o.Detail != c.detail {
				t.Errorf("said %s: %s; want %s: %s", o.Action, o.Detail, c.action, c.detail)
			}
			if c.unit == nil && !m.did("systemctl show unit.service") {
				t.Errorf("the machine was never asked whether the unit is there: %v", m.asked)
			}
			if m.did("systemctl start") || m.did("systemctl enable") {
				t.Errorf("something was started on the way out: %v", m.asked)
			}
			if _, still := after.Find("s"); still {
				t.Error("still recorded")
			}
		})
	}
}

func TestAUnitTheMeshStartedIsStoppedWhenUndeclaredAndOneFoundRunningIsNot(t *testing.T) {
	// End to end: found by the first apply, carried, given back.
	m := unitsMachine(map[string]*fakeUnit{
		"filter.service":  {active: "inactive", enabled: "disabled"},
		"runtime.service": {active: "active", enabled: "enabled"},
	})
	_, state, err := applyOn(t, `{"declaration":1,"resources":[
	  {"id":"filter","type":"service","unit":"filter.service","state":"running","boot":"enabled"},
	  {"id":"runtime","type":"service","unit":"runtime.service","state":"running","boot":"enabled"}]}`,
		store.State{}, m)
	if err != nil {
		t.Fatal(err)
	}
	if m.units["filter.service"].active != "active" {
		t.Fatal("the fixture did not start the filter")
	}
	if _, _, err := applyOn(t, nothingButA(t), state, m); err != nil {
		t.Fatal(err)
	}
	if u := m.units["filter.service"]; u.active != "inactive" || u.enabled != "disabled" {
		t.Errorf("the filter the mesh started and enabled is %s and %s", u.active, u.enabled)
	}
	if u := m.units["runtime.service"]; u.active != "active" || u.enabled != "enabled" {
		t.Errorf("the runtime that was running before the mesh is %s and %s", u.active, u.enabled)
	}
}

func TestAUnitHeldAndThenTakenIsFoundAsThePredecessorLeftIt(t *testing.T) {
	// Held while its module was untaken, nothing was applied and nothing found; taken, the first
	// apply finds the predecessor's unit — stopped, but started at boot — before starting it.
	dir := t.TempDir()
	m := unitsMachine(map[string]*fakeUnit{"hello.service": {active: "inactive", enabled: "enabled"}})
	service := `{"id":"hello-web.unit","type":"service","unit":"hello.service","state":"running","boot":"enabled"}`
	_, held := applyAdopted(t, adopted(t, untaken("hello-web.unit"), service), store.State{}, m, dir)
	if _, ok := held.HeldAt("hello-web.unit"); !ok {
		t.Fatal("the fixture's unit was not held")
	}
	_, taken := applyAdopted(t, adopted(t, `{"taken":["hello-web"]}`, service), held, m, dir)
	rec, _ := taken.Find("hello-web.unit")
	if rec.Found == nil || rec.Found.State != "stopped" || rec.Found.Boot != "enabled" {
		t.Fatalf("found %+v, want the predecessor's stopped and enabled", rec.Found)
	}
	if m.units["hello.service"].active != "active" {
		t.Fatal("the taken unit was not started")
	}
	if _, _, err := applyOn(t, nothingButA(t), taken, m); err != nil {
		t.Fatal(err)
	}
	if u := m.units["hello.service"]; u.active != "inactive" || u.enabled != "enabled" {
		t.Errorf("given back as %s and %s; the predecessor left it stopped and enabled", u.active, u.enabled)
	}
}