Keep what each machine says of what it runs, raise it, and gate on it (hq ADR 0240, to-be 48 Phase A)
The gate judged a module by what the mesh saw from outside, so a container that crash-looped after it applied passed it. Each machine's node-engine now states the health of every long-running resource it runs; the controller keeps the newest statement per machine, raises module.<module>.<machine>.unhealthy on the second statement in a row, clears it on the first that does not say it, and the gate passes a module only when every long-running resource of it is stated healthy since the send. An engine that states nothing is judged as before.
This commit is contained in:
jochen
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
|
||||
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
|
||||
// healthy one clears; a condition from before the send clears at the new build's start; an older
|
||||
// statement is refused; and an engine that states nothing raises nothing.
|
||||
|
||||
var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func aStatement(at time.Time, states ...string) link.Health {
|
||||
h := link.Health{Contract: link.LivenessContract, At: at}
|
||||
for i, s := range states {
|
||||
r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
|
||||
State: s, Since: at}
|
||||
if i > 0 {
|
||||
r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
|
||||
}
|
||||
if s == link.StateUnhealthy {
|
||||
r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
|
||||
}
|
||||
h.Resources = append(h.Resources, r)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
k := conditionsFrom
|
||||
const key = "module.letta.anchor.unhealthy"
|
||||
openKeys := func() []string {
|
||||
t.Helper()
|
||||
list, err := k.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys []string
|
||||
for _, c := range list {
|
||||
keys = append(keys, c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// One statement: nothing raised, and `node show` lists it as unconfirmed.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement raised %v", keys)
|
||||
}
|
||||
kept, had, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil || !had {
|
||||
t.Fatalf("the statement was not kept: %v %v", had, err)
|
||||
}
|
||||
if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
|
||||
!strings.Contains(lines, "letta.server") {
|
||||
t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
|
||||
}
|
||||
|
||||
// The second in a row raises it — the module's own, never the other module's on the machine.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
|
||||
h0.Add(2*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 1 || keys[0] != key {
|
||||
t.Fatalf("two statements raised %v, not %s", keys, key)
|
||||
}
|
||||
c, _, _ := k.Get(ctx, key)
|
||||
if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
|
||||
!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
|
||||
!strings.Contains(c.Evidence[0].Said, "letta-server") {
|
||||
t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
|
||||
}
|
||||
|
||||
// Standing four hours, it is urgent.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
|
||||
time.Now().Add(5*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("unhealthy for four hours is still %s", c.Severity)
|
||||
}
|
||||
|
||||
// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
|
||||
h0.Add(6*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
|
||||
}
|
||||
// And the streak starts again: one unhealthy statement after it raises nothing.
|
||||
if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys := openKeys(); len(keys) != 0 {
|
||||
t.Fatalf("one statement after a clearing raised %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOlderHealthStatementIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
before := healthRefused.Load()
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An event said before the report that overtook it arrives late.
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _, err := inv.HealthOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
|
||||
t.Fatalf("the older statement replaced the newer: %+v", kept)
|
||||
}
|
||||
if healthRefused.Load() != before+1 {
|
||||
t.Fatalf("the refusal was not counted")
|
||||
}
|
||||
}
|
||||
|
||||
// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
|
||||
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
|
||||
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
|
||||
t.Fatalf("health kept for an engine that said none: %v %v", had, err)
|
||||
}
|
||||
if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
|
||||
t.Fatalf("node show: %v", lines)
|
||||
}
|
||||
// And one that does, through the report, is kept.
|
||||
h := aStatement(time.Now().UTC(), link.StateHealthy)
|
||||
if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
|
||||
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user