//go:build integration package main import ( "strings" "testing" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is // checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a // healthy one clears; a condition from before the send clears at the new build's start; an older // statement is refused; and an engine that states nothing raises nothing. var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) func aStatement(at time.Time, states ...string) link.Health { h := link.Health{Contract: link.LivenessContract, At: at} for i, s := range states { r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server", State: s, Since: at} if i > 0 { r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service" } if s == link.StateUnhealthy { r.Reason, r.Restarts, r.Streak = "restarting", 4, 2 } h.Resources = append(h.Resources, r) } return h } func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory k := conditionsFrom const key = "module.letta.anchor.unhealthy" openKeys := func() []string { t.Helper() list, err := k.Open(ctx) if err != nil { t.Fatal(err) } var keys []string for _, c := range list { keys = append(keys, c.Key) } return keys } if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil { t.Fatal(err) } // One statement: nothing raised, and `node show` lists it as unconfirmed. if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy), h0.Add(time.Minute)); err != nil { t.Fatal(err) } if keys := openKeys(); len(keys) != 0 { t.Fatalf("one statement raised %v", keys) } kept, had, err := inv.HealthOf(ctx, "anchor") if err != nil || !had { t.Fatalf("the statement was not kept: %v %v", had, err) } if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") || !strings.Contains(lines, "letta.server") { t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines) } // The second in a row raises it — the module's own, never the other module's on the machine. if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy), h0.Add(2*time.Minute)); err != nil { t.Fatal(err) } if keys := openKeys(); len(keys) != 1 || keys[0] != key { t.Fatalf("two statements raised %v, not %s", keys, key) } c, _, _ := k.Get(ctx, key) if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" || !strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") || !strings.Contains(c.Evidence[0].Said, "letta-server") { t.Fatalf("the condition does not say it in words with its evidence: %+v", c) } // Standing four hours, it is urgent. if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy), time.Now().Add(5*time.Hour)); err != nil { t.Fatal(err) } if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent { t.Fatalf("unhealthy for four hours is still %s", c.Severity) } // A new build's start — every start begins in `starting` — clears it: what follows is the new build's. if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy), h0.Add(6*time.Hour)); err != nil { t.Fatal(err) } if keys := openKeys(); len(keys) != 0 { t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys) } // And the streak starts again: one unhealthy statement after it raises nothing. if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil { t.Fatal(err) } if keys := openKeys(); len(keys) != 0 { t.Fatalf("one statement after a clearing raised %v", keys) } } func TestAnOlderHealthStatementIsRefused(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory before := healthRefused.Load() if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil { t.Fatal(err) } // An event said before the report that overtook it arrives late. if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil { t.Fatal(err) } kept, _, err := inv.HealthOf(ctx, "anchor") if err != nil { t.Fatal(err) } if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy { t.Fatalf("the older statement replaced the newer: %+v", kept) } if healthRefused.Load() != before+1 { t.Fatalf("the refusal was not counted") } } // **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the // gate judges its machine as before — never healthy for having said nothing, never unhealthy. func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) { open := aMesh(t) ctx := t.Context() l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}} if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil { t.Fatal(err) } if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had { t.Fatalf("health kept for an engine that said none: %v %v", had, err) } if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") { t.Fatalf("node show: %v", lines) } // And one that does, through the report, is kept. h := aStatement(time.Now().UTC(), link.StateHealthy) if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil { t.Fatal(err) } if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 { t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err) } } // A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240 // Phase B): an older engine is strict and would refuse the whole declaration for the field. func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory reads := func() bool { t.Helper() got, err := engineReadsHealth(ctx, inv, "anchor") if err != nil { t.Fatal(err) } return got } if reads() { t.Fatal("an engine that never stated anything is sent health") } if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil { t.Fatal(err) } if reads() { t.Fatal("an engine judging liveness alone is sent health") } later := aStatement(h0.Add(time.Minute), link.StateHealthy) later.Contract = link.ReadinessContract if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil { t.Fatal(err) } if !reads() { t.Fatal("an engine that reads health is not sent it") } }