jochen
1fc1e74cc3
mesh/merge-gate pass: builds mesh-host → ace, g14, novox, shanks; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group feat/a-module-says-how-it-is-healthy delivered: every member is delivered
A container that crash-looped after its compose applied passed every check the gate had: nothing looked at what a module runs. The node-engine now judges every long-running resource on every look — one read of the runtime, one per service manager — keeps the restarts it counts across recreates and its own restarts, and says the state in every report and as an event on change, again every minute while not healthy. It reads only; nothing is restarted for being unhealthy.
333 lines
13 KiB
Go
333 lines
13 KiB
Go
package liveness
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
)
|
|
|
|
// The judge, over a fake runtime and service manager (novox/hq ADR 0240, "how it is checked", rule 1):
|
|
// a container recreated keeps its counted restarts, and so does the engine restarted; a restart inside
|
|
// grace is not counted; two restarts within the settle window after grace make it unhealthy; a resource
|
|
// under a maintenance step is held.
|
|
|
|
// fakeRuntime is what a look reads, set by the test.
|
|
type fakeRuntime struct {
|
|
containers map[string]Observed
|
|
units map[string]Observed
|
|
err error
|
|
}
|
|
|
|
func (f *fakeRuntime) Containers(_ context.Context, names []string) (map[string]Observed, error) {
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
out := map[string]Observed{}
|
|
for _, n := range names {
|
|
if o, ok := f.containers[n]; ok {
|
|
out[n] = o
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRuntime) Units(_ context.Context, _, _ string, units []string) (map[string]Observed, error) {
|
|
out := map[string]Observed{}
|
|
for _, u := range units {
|
|
out[u] = f.units[u]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// clock is a time the test moves.
|
|
type clock struct{ now time.Time }
|
|
|
|
func (c *clock) Now() time.Time { return c.now }
|
|
|
|
var t0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
|
|
|
func aJudge(t *testing.T, rt Runtime, c *clock, path string) *Judge {
|
|
t.Helper()
|
|
j, err := Open(path, rt)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
j.Now = c.Now
|
|
return j
|
|
}
|
|
|
|
var server = Resource{Module: "letta", ID: "letta.server", Kind: KindContainer, Target: "letta-server"}
|
|
|
|
func running(id string, restarts int64, started string) Observed {
|
|
return Observed{Found: true, Identity: id, Running: true, Restarts: restarts, Started: started}
|
|
}
|
|
|
|
func stateOf(t *testing.T, st Statement, id string) State {
|
|
t.Helper()
|
|
for _, r := range st.Resources {
|
|
if r.ID == id {
|
|
return r
|
|
}
|
|
}
|
|
t.Fatalf("%s is not in the statement: %+v", id, st)
|
|
return State{}
|
|
}
|
|
|
|
func TestARestartInsideGraceIsNotCountedAndTwoAfterItAreUnhealthy(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
j.Set([]Resource{server})
|
|
|
|
st, changed := j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Starting || !changed {
|
|
t.Fatalf("a fresh start is starting: %+v", s)
|
|
}
|
|
|
|
// Churn that stops (issue 058): restarted inside its grace, then up.
|
|
c.now = t0.Add(20 * time.Second)
|
|
rt.containers["letta-server"] = running("c1", 2, "b")
|
|
j.Look(t.Context())
|
|
c.now = t0.Add(70 * time.Second)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 0 {
|
|
t.Fatalf("restarts inside grace counted, or not healthy after it: %+v", s)
|
|
}
|
|
|
|
// One restart after grace is not yet a crash loop.
|
|
c.now = t0.Add(2 * time.Minute)
|
|
rt.containers["letta-server"] = running("c1", 3, "c")
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 1 {
|
|
t.Fatalf("one restart after grace: %+v", s)
|
|
}
|
|
|
|
// A second inside the settle window is.
|
|
c.now = t0.Add(3 * time.Minute)
|
|
rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarting: true, Restarts: 4, Started: "d"}
|
|
st, changed = j.Look(t.Context())
|
|
s := stateOf(t, st, "letta.server")
|
|
if s.State != Unhealthy || s.Reason != ReasonRestarting || s.Restarts != 2 || s.Streak != 1 || !changed {
|
|
t.Fatalf("two restarts within the settle window after grace: %+v", s)
|
|
}
|
|
c.now = t0.Add(3*time.Minute + 15*time.Second)
|
|
st, changed = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.Streak != 2 || changed || !s.Since.Equal(t0.Add(3*time.Minute)) {
|
|
t.Fatalf("the streak and since of a state that holds: %+v (changed %v)", s, changed)
|
|
}
|
|
|
|
// Past the settle window with no restart, it is alive again.
|
|
c.now = t0.Add(14 * time.Minute)
|
|
rt.containers["letta-server"] = running("c1", 4, "d")
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 {
|
|
t.Fatalf("a crash loop that stopped ten minutes ago: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestARecreatedContainerKeepsItsCountedRestartsAndStartsAgain(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
j.Set([]Resource{server})
|
|
j.Look(t.Context())
|
|
for i, at := range []time.Duration{2 * time.Minute, 3 * time.Minute} {
|
|
c.now = t0.Add(at)
|
|
rt.containers["letta-server"] = running("c1", int64(i+1), "x"+at.String())
|
|
j.Look(t.Context())
|
|
}
|
|
|
|
// The apply recreates it: the runtime's count is back to nothing, the engine's is not.
|
|
c.now = t0.Add(4 * time.Minute)
|
|
rt.containers["letta-server"] = running("c2", 0, "new")
|
|
st, _ := j.Look(t.Context())
|
|
s := stateOf(t, st, "letta.server")
|
|
if s.State != Starting || s.Restarts != 2 {
|
|
t.Fatalf("a recreate is a new start, with its count kept: %+v", s)
|
|
}
|
|
c.now = t0.Add(6 * time.Minute)
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Healthy || s.Restarts != 2 {
|
|
t.Fatalf("the new build, up after its grace, is healthy — the old build's restarts are not its: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestTheEngineRestartedKeepsWhatItCounted(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), FileName)
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
|
|
j := aJudge(t, rt, c, path)
|
|
j.Set([]Resource{server})
|
|
j.Look(t.Context())
|
|
c.now = t0.Add(2 * time.Minute)
|
|
rt.containers["letta-server"] = running("c1", 1, "b")
|
|
j.Look(t.Context())
|
|
|
|
// Another engine — this one restarted, or its successor — reads the file and goes on.
|
|
again := aJudge(t, rt, c, path)
|
|
c.now = t0.Add(2*time.Minute + 30*time.Second)
|
|
rt.containers["letta-server"] = running("c1", 2, "c")
|
|
st, _ := again.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Restarts != 2 {
|
|
t.Fatalf("the restarted engine forgot what it counted: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAContainerHeldByAWindowIsHeldAndJudgedAfreshAfter(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{"letta-server": running("c1", 0, "a")}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
windowOpen := true
|
|
j.HeldNow = func(time.Time) map[string]bool { return map[string]bool{"letta-server": windowOpen} }
|
|
j.Set([]Resource{server})
|
|
c.now = t0.Add(5 * time.Minute)
|
|
rt.containers["letta-server"] = Observed{Found: true, Identity: "c1", Restarts: 0, Started: "a"}
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Held {
|
|
t.Fatalf("a container a window holds still is held, neither alive nor dead: %+v", s)
|
|
}
|
|
windowOpen = false
|
|
rt.containers["letta-server"] = running("c1", 0, "after")
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Starting {
|
|
t.Fatalf("the window ended is a start, judged from a fresh grace: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestDownAfterGraceAndUnknownWhenNothingCouldBeRead(t *testing.T) {
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{containers: map[string]Observed{}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
j.Set([]Resource{server})
|
|
if s := stateOf(t, func() Statement { st, _ := j.Look(t.Context()); return st }(), "letta.server"); s.State != Starting {
|
|
t.Fatalf("not there at its first look is still in its grace: %+v", s)
|
|
}
|
|
c.now = t0.Add(2 * time.Minute)
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Unhealthy || s.Reason != ReasonDown {
|
|
t.Fatalf("not there after its grace is down: %+v", s)
|
|
}
|
|
rt.err = errors.New("cannot connect to the runtime")
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "letta.server"); s.State != Unknown || !strings.Contains(s.Reason, "cannot connect") {
|
|
t.Fatalf("a runtime that does not answer is unknown, never down: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestAUnitRestartedByItsManagerIsCountedAndOneStartedAgainIsNot(t *testing.T) {
|
|
unit := Resource{Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"}
|
|
c := &clock{now: t0}
|
|
rt := &fakeRuntime{units: map[string]Observed{"mosquitto.service": running("i1", 0, "")}}
|
|
j := aJudge(t, rt, c, filepath.Join(t.TempDir(), FileName))
|
|
j.Set([]Resource{unit})
|
|
j.Look(t.Context())
|
|
c.now = t0.Add(2 * time.Minute)
|
|
rt.units["mosquitto.service"] = running("i2", 1, "") // the manager's Restart=
|
|
j.Look(t.Context())
|
|
c.now = t0.Add(3 * time.Minute)
|
|
rt.units["mosquitto.service"] = running("i3", 0, "") // restarted by the apply: NRestarts reset
|
|
st, _ := j.Look(t.Context())
|
|
if s := stateOf(t, st, "mqtt.broker"); s.State != Starting || s.Restarts != 1 {
|
|
t.Fatalf("a restart somebody made is a new start, the manager's is counted: %+v", s)
|
|
}
|
|
c.now = t0.Add(5 * time.Minute)
|
|
rt.units["mosquitto.service"] = Observed{Found: true, Identity: "", Running: false}
|
|
st, _ = j.Look(t.Context())
|
|
if s := stateOf(t, st, "mqtt.broker"); s.State != Unhealthy || s.Reason != ReasonDown {
|
|
t.Fatalf("an inactive unit stated running is down: %+v", s)
|
|
}
|
|
}
|
|
|
|
// The long-running resources of a declaration: what stays up, by module; never a step, a schedule, a
|
|
// service whose lifecycle is the machine's, the mesh's own resources, or what an adopted machine holds.
|
|
func TestWhatIsLongRunning(t *testing.T) {
|
|
const image = "docker.io/library/postgres@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
d, err := declaration.ParseTrusted([]byte(`{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"mesh-store","image":"` + image + `"},
|
|
{"id":"letta.server","type":"container","name":"letta-server","image":"` + image + `"},
|
|
{"id":"letta.migrate","type":"container","name":"letta-migrate","image":"` + image + `","run-once":true},
|
|
{"id":"letta.sweep","type":"container","name":"letta-sweep","image":"` + image + `","schedule":"0 3 * * *"},
|
|
{"id":"novox.be.web","type":"container","name":"novox-web","image":"` + image + `"},
|
|
{"id":"mqtt.broker","type":"service","unit":"mosquitto.service","state":"running"},
|
|
{"id":"uplink.nm","type":"service","unit":"NetworkManager.service","restart-on":["mqtt.broker"]},
|
|
{"id":"found.thing","type":"container","name":"found","image":"` + image + `"}
|
|
]}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := LongRunning(d, map[string]bool{"found.thing": true})
|
|
var ids []string
|
|
for _, r := range got {
|
|
ids = append(ids, r.Module+"/"+r.ID)
|
|
}
|
|
want := "letta/letta.server novox.be/novox.be.web mqtt/mqtt.broker"
|
|
if strings.Join(ids, " ") != want {
|
|
t.Fatalf("long-running: %v, want %s", ids, want)
|
|
}
|
|
}
|
|
|
|
// **Nothing is restarted for being unhealthy** (ADR 0240 rule 6): a crash-looping container is judged
|
|
// unhealthy, and everything the judge asked the runtime and the manager was a read.
|
|
func TestAnUnhealthyContainerIsNeitherRestartedRecreatedNorStopped(t *testing.T) {
|
|
var asked []string
|
|
restarts := 0
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
asked = append(asked, name+" "+strings.Join(args, " "))
|
|
switch {
|
|
case name == "docker" && len(args) > 0 && args[0] == "version":
|
|
return "27.0.0\n", nil
|
|
case name == "docker":
|
|
restarts++
|
|
return "/letta-server\tc1\trestarting\t" + string(rune('0'+restarts)) + "\t2026-10-07T12:00:00Z\n", nil
|
|
case name == "systemctl":
|
|
return "Id=mosquitto.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nNRestarts=5\nInvocationID=\n", nil
|
|
}
|
|
return "", errors.New("not a command the judge may run")
|
|
}
|
|
c := &clock{now: t0}
|
|
j := aJudge(t, &Exec{Run: run}, c, filepath.Join(t.TempDir(), FileName))
|
|
j.Set([]Resource{server, {Module: "mqtt", ID: "mqtt.broker", Kind: KindService, Target: "mosquitto.service"}})
|
|
var st Statement
|
|
for i := 0; i < 6; i++ {
|
|
c.now = t0.Add(time.Duration(i) * time.Minute)
|
|
st, _ = j.Look(t.Context())
|
|
}
|
|
if s := stateOf(t, st, "letta.server"); s.State != Unhealthy {
|
|
t.Fatalf("the crash loop was not judged unhealthy: %+v", s)
|
|
}
|
|
for _, a := range asked {
|
|
read := strings.HasPrefix(a, "docker version ") || strings.HasPrefix(a, "docker container inspect ") ||
|
|
strings.HasPrefix(a, "systemctl show ")
|
|
if !read {
|
|
t.Errorf("the judge asked something that is not a read: %q", a)
|
|
}
|
|
}
|
|
}
|
|
|
|
// One read of every container per look, never one per container (ADR 0240: the engine stays cheap).
|
|
func TestOneLookIsOneReadOfEveryContainer(t *testing.T) {
|
|
inspects := 0
|
|
run := func(_ context.Context, name string, args ...string) (string, error) {
|
|
if args[0] == "container" {
|
|
inspects++
|
|
return "/a\tc1\trunning\t0\tx\n/b\tc2\trunning\t0\tx\n", errors.New("docker exited 1: Error: No such container: c")
|
|
}
|
|
return "27\n", nil
|
|
}
|
|
j := aJudge(t, &Exec{Run: run}, &clock{now: t0}, "")
|
|
j.Set([]Resource{{Module: "m", ID: "m.a", Kind: KindContainer, Target: "a"},
|
|
{Module: "m", ID: "m.b", Kind: KindContainer, Target: "b"}, {Module: "m", ID: "m.c", Kind: KindContainer, Target: "c"}})
|
|
st, _ := j.Look(t.Context())
|
|
if inspects != 1 {
|
|
t.Fatalf("%d inspects for one look", inspects)
|
|
}
|
|
if s := stateOf(t, st, "m.b"); s.State != Starting {
|
|
t.Fatalf("a container the inspect found, beside one it did not: %+v", s)
|
|
}
|
|
}
|