mesh-host/internal/liveness/liveness_test.go

package liveness

import (
	"context"
	"errors"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// The judge, over a fake runtime and service manager (novox/hq ADR 0240, "how it is checked", rule 1):
// a container recreated keeps its counted restarts, and so does the engine restarted; a restart inside
// grace is not counted; two restarts within the settle window after grace make it unhealthy; a resource
// under a maintenance step is held.

// fakeRuntime is what a look reads, set by the test.
type fakeRuntime struct {
	containers map[string]Observed
	units      map[string]Observed
	err        error
}

func (f *fakeRuntime) Containers(_ context.Context, names []string) (map[string]Observed, error) {
	if f.err != nil {
		return nil, f.err
	}
	out := map[string]Observed{}
	for _, n := range names {
		if o, ok := f.containers[n]; ok {
			out[n] = o
		}
	}
	return out, nil
}

func (f *fakeRuntime) Units(_ context.Context, _, _ string, units []string) (map[string]Observed, error) {
	out := map[string]Observed{}
	for _, u := range units {
		out[u] = f.units[u]
	}
	return out, nil
}

// clock is a time the test moves.
type clock struct{ now time.Time }

func (c *clock) Now() time.Time { return c.now }

var t0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)

func aJudge(t *testing.T, rt Runtime, c *clock, path string) *Judge {
	t.Helper()
	j, err := Open(path, rt)
	if err != nil {
		t.Fatal(err)
	}
	j.Now = c.Now
	return j
}

var server = Resource{Module: "letta", ID: "letta.server", Kind: KindContainer, Target: "letta-server"}

func running(id string, restarts int64, started string) Observed {
	return Observed{Found: true, Identity: id, Running: true, Restarts: restarts, Started: started}
}

func stateOf(t *testing.T, st Statement, id string) State {
	t.Helper()
	for _, r := range st.Resources {
		if r.ID == id {
			return r
		}
	}
	t.Fatalf("%s is not in the statement: %+v", id, st)
	return State{}
}

func TestARestartInsideGraceIsNotCountedAndTwoAfterItAreUnhealthy(t *testing.T) {
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	j.Set([]Resource{server})

	st, changed := j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Starting || !changed {
		t.Fatalf("a fresh start is starting: %+v", s)
	}

	// Churn that stops (issue 058): restarted inside its grace, then up.
	c.now = t0.Add(20 * time.Second)
	rt.containers["letta-server"] = running("c1", 2, "b")
	j.Look(t.Context())
	c.now = t0.Add(70 * time.Second)
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 0 {
		t.Fatalf("restarts inside grace counted, or not healthy after it: %+v", s)
	}

	// One restart after grace is not yet a crash loop.
	c.now = t0.Add(2 * time.Minute)
	rt.containers["letta-server"] = running("c1", 3, "c")
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 1 {
		t.Fatalf("one restart after grace: %+v", s)
	}

	// A second inside the settle window is.
	c.now = t0.Add(3 * time.Minute)
	rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarting: true, Restarts: 4, Started: "d"}
	st, changed = j.Look(t.Context())
	s := stateOf(t, st, "letta.server")
	if s.State != Unhealthy || s.Reason != ReasonRestarting || s.Restarts != 2 || s.Streak != 1 || !changed {
		t.Fatalf("two restarts within the settle window after grace: %+v", s)
	}
	c.now = t0.Add(3*time.Minute + 15*time.Second)
	st, changed = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.Streak != 2 || changed || !s.Since.Equal(t0.Add(3*time.Minute)) {
		t.Fatalf("the streak and since of a state that holds: %+v (changed %v)", s, changed)
	}

	// Past the settle window with no restart, it is alive again.
	c.now = t0.Add(14 * time.Minute)
	rt.containers["letta-server"] = running("c1", 4, "d")
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 {
		t.Fatalf("a crash loop that stopped ten minutes ago: %+v", s)
	}
}

func TestARecreatedContainerKeepsItsCountedRestartsAndStartsAgain(t *testing.T) {
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	j.Set([]Resource{server})
	j.Look(t.Context())
	for i, at := range []time.Duration{2 * time.Minute, 3 * time.Minute} {
		c.now = t0.Add(at)
		rt.containers["letta-server"] = running("c1", int64(i+1), "x"+at.String())
		j.Look(t.Context())
	}

	// The apply recreates it: the runtime's count is back to nothing, the engine's is not.
	c.now = t0.Add(4 * time.Minute)
	rt.containers["letta-server"] = running("c2", 0, "new")
	st, _ := j.Look(t.Context())
	s := stateOf(t, st, "letta.server")
	if s.State != Starting || s.Restarts != 2 {
		t.Fatalf("a recreate is a new start, with its count kept: %+v", s)
	}
	c.now = t0.Add(6 * time.Minute)
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 {
		t.Fatalf("the new build, up after its grace, is healthy — the old build's restarts are not its: %+v", s)
	}
}

func TestTheEngineRestartedKeepsWhatItCounted(t *testing.T) {
	path := filepath.Join(t.TempDir(), FileName)
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
	j := aJudge(t, rt, c, path)
	j.Set([]Resource{server})
	j.Look(t.Context())
	c.now = t0.Add(2 * time.Minute)
	rt.containers["letta-server"] = running("c1", 1, "b")
	j.Look(t.Context())

	// Another engine — this one restarted, or its successor — reads the file and goes on.
	again := aJudge(t, rt, c, path)
	c.now = t0.Add(2*time.Minute + 30*time.Second)
	rt.containers["letta-server"] = running("c1", 2, "c")
	st, _ := again.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Restarts != 2 {
		t.Fatalf("the restarted engine forgot what it counted: %+v", s)
	}
}

func TestAContainerHeldByAWindowIsHeldAndJudgedAfreshAfter(t *testing.T) {
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	windowOpen := true
	j.HeldNow = func(time.Time) map[string]bool { return map[string]bool{"letta-server": windowOpen} }
	j.Set([]Resource{server})
	c.now = t0.Add(5 * time.Minute)
	rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarts: 0, Started: "a"}
	st, _ := j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Held {
		t.Fatalf("a container a window holds still is held, neither alive nor dead: %+v", s)
	}
	windowOpen = false
	rt.containers["letta-server"] = running("c1", 0, "after")
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Starting {
		t.Fatalf("the window ended is a start, judged from a fresh grace: %+v", s)
	}
}

func TestDownAfterGraceAndUnknownWhenNothingCouldBeRead(t *testing.T) {
	c := &clock{now: t0}
	rt := &fakeRuntime{containers: map[string]Observed{}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	j.Set([]Resource{server})
	if s := stateOf(t, func() Statement { st, _ := j.Look(t.Context()); return st }(), "letta.server"); s.State != Starting {
		t.Fatalf("not there at its first look is still in its grace: %+v", s)
	}
	c.now = t0.Add(2 * time.Minute)
	st, _ := j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Reason != ReasonDown {
		t.Fatalf("not there after its grace is down: %+v", s)
	}
	rt.err = errors.New("cannot connect to the runtime")
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "letta.server"); s.State != Unknown || !strings.Contains(s.Reason, "cannot connect") {
		t.Fatalf("a runtime that does not answer is unknown, never down: %+v", s)
	}
}

func TestAUnitRestartedByItsManagerIsCountedAndOneStartedAgainIsNot(t *testing.T) {
	unit := Resource{Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"}
	c := &clock{now: t0}
	rt := &fakeRuntime{units: map[string]Observed{"mosquitto.service": running("i1", 0, "")}}
	j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
	j.Set([]Resource{unit})
	j.Look(t.Context())
	c.now = t0.Add(2 * time.Minute)
	rt.units["mosquitto.service"] = running("i2", 1, "") // the manager's Restart=
	j.Look(t.Context())
	c.now = t0.Add(3 * time.Minute)
	rt.units["mosquitto.service"] = running("i3", 0, "") // restarted by the apply: NRestarts reset
	st, _ := j.Look(t.Context())
	if s := stateOf(t, st, "mqtt.broker"); s.State != Starting || s.Restarts != 1 {
		t.Fatalf("a restart somebody made is a new start, the manager's is counted: %+v", s)
	}
	c.now = t0.Add(5 * time.Minute)
	rt.units["mosquitto.service"] = Observed{Found: true, Identity: "", Running: false}
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "mqtt.broker"); s.State != Unhealthy || s.Reason != ReasonDown {
		t.Fatalf("an inactive unit stated running is down: %+v", s)
	}
}

// The long-running resources of a declaration: what stays up, by module; never a step, a schedule, a
// service whose lifecycle is the machine's, the mesh's own resources, or what an adopted machine holds.
func TestWhatIsLongRunning(t *testing.T) {
	const image = "docker.io/library/postgres@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
	d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[
	  {"id":"store","type":"container","name":"mesh-store","image":"` + image + `"},
	  {"id":"letta.server","type":"container","name":"letta-server","image":"` + image + `"},
	  {"id":"letta.migrate","type":"container","name":"letta-migrate","image":"` + image + `","run-once":true},
	  {"id":"letta.sweep","type":"container","name":"letta-sweep","image":"` + image + `","schedule":"0 3 * * *"},
	  {"id":"novox.be.web","type":"container","name":"novox-web","image":"` + image + `"},
	  {"id":"mqtt.broker","type":"service","unit":"mosquitto.service","state":"running"},
	  {"id":"uplink.nm","type":"service","unit":"NetworkManager.service","restart-on":["mqtt.broker"]},
	  {"id":"found.thing","type":"container","name":"found","image":"` + image + `"}
	]}`))
	if err != nil {
		t.Fatal(err)
	}
	got := LongRunning(d, map[string]bool{"found.thing": true})
	var ids []string
	for _, r := range got {
		ids = append(ids, r.Module+"/"+r.ID)
	}
	want := "letta/letta.server novox.be/novox.be.web mqtt/mqtt.broker"
	if strings.Join(ids, " ") != want {
		t.Fatalf("long-running: %v, want %s", ids, want)
	}
}

// **Nothing is restarted for being unhealthy** (ADR 0240 rule 6): a crash-looping container is judged
// unhealthy, and everything the judge asked the runtime and the manager was a read.
func TestAnUnhealthyContainerIsNeitherRestartedRecreatedNorStopped(t *testing.T) {
	var asked []string
	restarts := 0
	run := func(_ context.Context, name string, args ...string) (string, error) {
		asked = append(asked, name+" "+strings.Join(args, " "))
		switch {
		case name == "docker" && len(args) > 0 && args[0] == "version":
			return "27.0.0\n", nil
		case name == "docker":
			restarts++
			return "/letta-server\tc1\trestarting\t" + string(rune('0'+restarts)) + "\t2026-10-07T12:00:00Z\n", nil
		case name == "systemctl":
			return "Id=mosquitto.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nNRestarts=5\nInvocationID=\n", nil
		}
		return "", errors.New("not a command the judge may run")
	}
	c := &clock{now: t0}
	j := aJudge(t, &Exec{Run: run}, c, filepath.Join(t.TempDir(), FileName))
	j.Set([]Resource{server, {Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"}})
	var st Statement
	for i := 0; i < 6; i++ {
		c.now = t0.Add(time.Duration(i) * time.Minute)
		st, _ = j.Look(t.Context())
	}
	if s := stateOf(t, st, "letta.server"); s.State != Unhealthy {
		t.Fatalf("the crash loop was not judged unhealthy: %+v", s)
	}
	for _, a := range asked {
		read := strings.HasPrefix(a, "docker version ") || strings.HasPrefix(a, "docker container inspect ") ||
			strings.HasPrefix(a, "systemctl show ")
		if !read {
			t.Errorf("the judge asked something that is not a read: %q", a)
		}
	}
}

// One read of every container per look, never one per container (ADR 0240: the engine stays cheap).
func TestOneLookIsOneReadOfEveryContainer(t *testing.T) {
	inspects := 0
	run := func(_ context.Context, name string, args ...string) (string, error) {
		if args[0] == "container" {
			inspects++
			return "/a\tc1\trunning\t0\tx\n/b\tc2\trunning\t0\tx\n", errors.New("docker exited 1: Error: No such container: c")
		}
		return "27\n", nil
	}
	j := aJudge(t, &Exec{Run: run}, &clock{now: t0}, "")
	j.Set([]Resource{{Module: "m", ID: "m.a", Kind: KindContainer, Target: "a"},
		{Module: "m", ID: "m.b", Kind: KindContainer, Target: "b"}, {Module: "m", ID: "m.c", Kind: KindContainer, Target: "c"}})
	st, _ := j.Look(t.Context())
	if inspects != 1 {
		t.Fatalf("%d inspects for one look", inspects)
	}
	if s := stateOf(t, st, "m.b"); s.State != Starting {
		t.Fatalf("a container the inspect found, beside one it did not: %+v", s)
	}
}

// **A look starts no account's manager** (novox/hq issue 764, ADR 0177): a user-scoped unit's manager is
// asked of the machine's first; one not running is not read, nothing is started, and its unit is said
// unknown with why — away, never down; one running is read as before.
func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) {
	var asked []string
	manager := "inactive"
	run := func(_ context.Context, name string, args ...string) (string, error) {
		asked = append(asked, name+" "+strings.Join(args, " "))
		switch {
		case name == "getent":
			return "operator:x:1000:1000::/home/operator:/bin/zsh\n", nil
		case name == "systemctl" && len(args) > 0 && args[0] == "is-active":
			if manager != "active" {
				return manager + "\n", errors.New("systemctl exited 3")
			}
			return "active\n", nil
		case name == "systemctl":
			return "Id=dunst.service\nLoadState=loaded\nActiveState=active\nSubState=running\nNRestarts=0\nInvocationID=i1\n", nil
		}
		return "", errors.New("not a command the judge may run")
	}
	c := &clock{now: t0}
	j := aJudge(t, &Exec{Run: run, System: "arch"}, c, "")
	j.Set([]Resource{{Module: "dunst", ID: "dunst.daemon", Kind: KindService, Target: "dunst.service",
		Scope: "user", User: "operator"}})
	var st Statement
	for i := 0; i < 4; i++ {
		c.now = t0.Add(time.Duration(i) * time.Minute)
		st, _ = j.Look(t.Context())
	}
	if s := stateOf(t, st, "dunst.daemon"); s.State != Unknown || !strings.Contains(s.Reason, "not logged in") {
		t.Fatalf("a unit in a manager not running is away, not down: %+v", s)
	}
	for _, a := range asked {
		if strings.Contains(a, "--machine=") {
			t.Fatalf("the account's own manager was asked while it was not running: %q", a)
		}
		if a != "getent passwd operator" && a != "systemctl is-active user@1000.service" {
			t.Errorf("asked something but whether the manager runs: %q", a)
		}
	}
	// Logged in: read in the account's own manager, as before.
	manager, asked = "active", nil
	c.now = t0.Add(5 * time.Minute)
	st, _ = j.Look(t.Context())
	if s := stateOf(t, st, "dunst.daemon"); s.State == Unknown || s.State == Unhealthy {
		t.Fatalf("a unit in a running manager is read: %+v", s)
	}
	if last := asked[len(asked)-1]; !strings.HasPrefix(last, "systemctl --user --machine=operator@ show ") {
		t.Fatalf("the running manager is read in the account's own: %q", last)
	}
}