package liveness import ( "context" "errors" "path/filepath" "strings" "testing" "time" "github.com/novox/mesh-host/internal/declaration" ) // The judge, over a fake runtime and service manager (novox/hq ADR 0240, "how it is checked", rule 1): // a container recreated keeps its counted restarts, and so does the engine restarted; a restart inside // grace is not counted; two restarts within the settle window after grace make it unhealthy; a resource // under a maintenance step is held. // fakeRuntime is what a look reads, set by the test. type fakeRuntime struct { containers map[string]Observed units map[string]Observed err error } func (f *fakeRuntime) Containers(_ context.Context, names []string) (map[string]Observed, error) { if f.err != nil { return nil, f.err } out := map[string]Observed{} for _, n := range names { if o, ok := f.containers[n]; ok { out[n] = o } } return out, nil } func (f *fakeRuntime) Units(_ context.Context, _, _ string, units []string) (map[string]Observed, error) { out := map[string]Observed{} for _, u := range units { out[u] = f.units[u] } return out, nil } // clock is a time the test moves. type clock struct{ now time.Time } func (c *clock) Now() time.Time { return c.now } var t0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) func aJudge(t *testing.T, rt Runtime, c *clock, path string) *Judge { t.Helper() j, err := Open(path, rt) if err != nil { t.Fatal(err) } j.Now = c.Now return j } var server = Resource{Module: "letta", ID: "letta.server", Kind: KindContainer, Target: "letta-server"} func running(id string, restarts int64, started string) Observed { return Observed{Found: true, Identity: id, Running: true, Restarts: restarts, Started: started} } func stateOf(t *testing.T, st Statement, id string) State { t.Helper() for _, r := range st.Resources { if r.ID == id { return r } } t.Fatalf("%s is not in the statement: %+v", id, st) return State{} } func TestARestartInsideGraceIsNotCountedAndTwoAfterItAreUnhealthy(t *testing.T) { c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) j.Set([]Resource{server}) st, changed := j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Starting || !changed { t.Fatalf("a fresh start is starting: %+v", s) } // Churn that stops (issue 058): restarted inside its grace, then up. c.now = t0.Add(20 * time.Second) rt.containers["letta-server"] = running("c1", 2, "b") j.Look(t.Context()) c.now = t0.Add(70 * time.Second) st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 0 { t.Fatalf("restarts inside grace counted, or not healthy after it: %+v", s) } // One restart after grace is not yet a crash loop. c.now = t0.Add(2 * time.Minute) rt.containers["letta-server"] = running("c1", 3, "c") st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 1 { t.Fatalf("one restart after grace: %+v", s) } // A second inside the settle window is. c.now = t0.Add(3 * time.Minute) rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarting: true, Restarts: 4, Started: "d"} st, changed = j.Look(t.Context()) s := stateOf(t, st, "letta.server") if s.State != Unhealthy || s.Reason != ReasonRestarting || s.Restarts != 2 || s.Streak != 1 || !changed { t.Fatalf("two restarts within the settle window after grace: %+v", s) } c.now = t0.Add(3*time.Minute + 15*time.Second) st, changed = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.Streak != 2 || changed || !s.Since.Equal(t0.Add(3*time.Minute)) { t.Fatalf("the streak and since of a state that holds: %+v (changed %v)", s, changed) } // Past the settle window with no restart, it is alive again. c.now = t0.Add(14 * time.Minute) rt.containers["letta-server"] = running("c1", 4, "d") st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 { t.Fatalf("a crash loop that stopped ten minutes ago: %+v", s) } } func TestARecreatedContainerKeepsItsCountedRestartsAndStartsAgain(t *testing.T) { c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) j.Set([]Resource{server}) j.Look(t.Context()) for i, at := range []time.Duration{2 * time.Minute, 3 * time.Minute} { c.now = t0.Add(at) rt.containers["letta-server"] = running("c1", int64(i+1), "x"+at.String()) j.Look(t.Context()) } // The apply recreates it: the runtime's count is back to nothing, the engine's is not. c.now = t0.Add(4 * time.Minute) rt.containers["letta-server"] = running("c2", 0, "new") st, _ := j.Look(t.Context()) s := stateOf(t, st, "letta.server") if s.State != Starting || s.Restarts != 2 { t.Fatalf("a recreate is a new start, with its count kept: %+v", s) } c.now = t0.Add(6 * time.Minute) st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 { t.Fatalf("the new build, up after its grace, is healthy — the old build's restarts are not its: %+v", s) } } func TestTheEngineRestartedKeepsWhatItCounted(t *testing.T) { path := filepath.Join(t.TempDir(), FileName) c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}} j := aJudge(t, rt, c, path) j.Set([]Resource{server}) j.Look(t.Context()) c.now = t0.Add(2 * time.Minute) rt.containers["letta-server"] = running("c1", 1, "b") j.Look(t.Context()) // Another engine — this one restarted, or its successor — reads the file and goes on. again := aJudge(t, rt, c, path) c.now = t0.Add(2*time.Minute + 30*time.Second) rt.containers["letta-server"] = running("c1", 2, "c") st, _ := again.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Restarts != 2 { t.Fatalf("the restarted engine forgot what it counted: %+v", s) } } func TestAContainerHeldByAWindowIsHeldAndJudgedAfreshAfter(t *testing.T) { c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) windowOpen := true j.HeldNow = func(time.Time) map[string]bool { return map[string]bool{"letta-server": windowOpen} } j.Set([]Resource{server}) c.now = t0.Add(5 * time.Minute) rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarts: 0, Started: "a"} st, _ := j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Held { t.Fatalf("a container a window holds still is held, neither alive nor dead: %+v", s) } windowOpen = false rt.containers["letta-server"] = running("c1", 0, "after") st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Starting { t.Fatalf("the window ended is a start, judged from a fresh grace: %+v", s) } } func TestDownAfterGraceAndUnknownWhenNothingCouldBeRead(t *testing.T) { c := &clock{now: t0} rt := &fakeRuntime{containers: map[string]Observed{}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) j.Set([]Resource{server}) if s := stateOf(t, func() Statement { st, _ := j.Look(t.Context()); return st }(), "letta.server"); s.State != Starting { t.Fatalf("not there at its first look is still in its grace: %+v", s) } c.now = t0.Add(2 * time.Minute) st, _ := j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Reason != ReasonDown { t.Fatalf("not there after its grace is down: %+v", s) } rt.err = errors.New("cannot connect to the runtime") st, _ = j.Look(t.Context()) if s := stateOf(t, st, "letta.server"); s.State != Unknown || !strings.Contains(s.Reason, "cannot connect") { t.Fatalf("a runtime that does not answer is unknown, never down: %+v", s) } } func TestAUnitRestartedByItsManagerIsCountedAndOneStartedAgainIsNot(t *testing.T) { unit := Resource{Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"} c := &clock{now: t0} rt := &fakeRuntime{units: map[string]Observed{"mosquitto.service": running("i1", 0, "")}} j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName)) j.Set([]Resource{unit}) j.Look(t.Context()) c.now = t0.Add(2 * time.Minute) rt.units["mosquitto.service"] = running("i2", 1, "") // the manager's Restart= j.Look(t.Context()) c.now = t0.Add(3 * time.Minute) rt.units["mosquitto.service"] = running("i3", 0, "") // restarted by the apply: NRestarts reset st, _ := j.Look(t.Context()) if s := stateOf(t, st, "mqtt.broker"); s.State != Starting || s.Restarts != 1 { t.Fatalf("a restart somebody made is a new start, the manager's is counted: %+v", s) } c.now = t0.Add(5 * time.Minute) rt.units["mosquitto.service"] = Observed{Found: true, Identity: "", Running: false} st, _ = j.Look(t.Context()) if s := stateOf(t, st, "mqtt.broker"); s.State != Unhealthy || s.Reason != ReasonDown { t.Fatalf("an inactive unit stated running is down: %+v", s) } } // The long-running resources of a declaration: what stays up, by module; never a step, a schedule, a // service whose lifecycle is the machine's, the mesh's own resources, or what an adopted machine holds. func TestWhatIsLongRunning(t *testing.T) { const image = "docker.io/library/postgres@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[ {"id":"store","type":"container","name":"mesh-store","image":"` + image + `"}, {"id":"letta.server","type":"container","name":"letta-server","image":"` + image + `"}, {"id":"letta.migrate","type":"container","name":"letta-migrate","image":"` + image + `","run-once":true}, {"id":"letta.sweep","type":"container","name":"letta-sweep","image":"` + image + `","schedule":"0 3 * * *"}, {"id":"novox.be.web","type":"container","name":"novox-web","image":"` + image + `"}, {"id":"mqtt.broker","type":"service","unit":"mosquitto.service","state":"running"}, {"id":"uplink.nm","type":"service","unit":"NetworkManager.service","restart-on":["mqtt.broker"]}, {"id":"found.thing","type":"container","name":"found","image":"` + image + `"} ]}`)) if err != nil { t.Fatal(err) } got := LongRunning(d, map[string]bool{"found.thing": true}) var ids []string for _, r := range got { ids = append(ids, r.Module+"/"+r.ID) } want := "letta/letta.server novox.be/novox.be.web mqtt/mqtt.broker" if strings.Join(ids, " ") != want { t.Fatalf("long-running: %v, want %s", ids, want) } } // **Nothing is restarted for being unhealthy** (ADR 0240 rule 6): a crash-looping container is judged // unhealthy, and everything the judge asked the runtime and the manager was a read. func TestAnUnhealthyContainerIsNeitherRestartedRecreatedNorStopped(t *testing.T) { var asked []string restarts := 0 run := func(_ context.Context, name string, args ...string) (string, error) { asked = append(asked, name+" "+strings.Join(args, " ")) switch { case name == "docker" && len(args) > 0 && args[0] == "version": return "27.0.0\n", nil case name == "docker": restarts++ return "/letta-server\tc1\trestarting\t" + string(rune('0'+restarts)) + "\t2026-10-07T12:00:00Z\n", nil case name == "systemctl": return "Id=mosquitto.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nNRestarts=5\nInvocationID=\n", nil } return "", errors.New("not a command the judge may run") } c := &clock{now: t0} j := aJudge(t, &Exec{Run: run}, c, filepath.Join(t.TempDir(), FileName)) j.Set([]Resource{server, {Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"}}) var st Statement for i := 0; i < 6; i++ { c.now = t0.Add(time.Duration(i) * time.Minute) st, _ = j.Look(t.Context()) } if s := stateOf(t, st, "letta.server"); s.State != Unhealthy { t.Fatalf("the crash loop was not judged unhealthy: %+v", s) } for _, a := range asked { read := strings.HasPrefix(a, "docker version ") || strings.HasPrefix(a, "docker container inspect ") || strings.HasPrefix(a, "systemctl show ") if !read { t.Errorf("the judge asked something that is not a read: %q", a) } } } // One read of every container per look, never one per container (ADR 0240: the engine stays cheap). func TestOneLookIsOneReadOfEveryContainer(t *testing.T) { inspects := 0 run := func(_ context.Context, name string, args ...string) (string, error) { if args[0] == "container" { inspects++ return "/a\tc1\trunning\t0\tx\n/b\tc2\trunning\t0\tx\n", errors.New("docker exited 1: Error: No such container: c") } return "27\n", nil } j := aJudge(t, &Exec{Run: run}, &clock{now: t0}, "") j.Set([]Resource{{Module: "m", ID: "m.a", Kind: KindContainer, Target: "a"}, {Module: "m", ID: "m.b", Kind: KindContainer, Target: "b"}, {Module: "m", ID: "m.c", Kind: KindContainer, Target: "c"}}) st, _ := j.Look(t.Context()) if inspects != 1 { t.Fatalf("%d inspects for one look", inspects) } if s := stateOf(t, st, "m.b"); s.State != Starting { t.Fatalf("a container the inspect found, beside one it did not: %+v", s) } }