mesh-controller/cmd/mesh-controller/module_health_test.go

//go:build integration

package main

import (
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/compose"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
// healthy one clears; a condition from before the send clears at the new build's start; an older
// statement is refused; and an engine that states nothing raises nothing.

var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)

func aStatement(at time.Time, states ...string) link.Health {
	h := link.Health{Contract: link.LivenessContract, At: at}
	for i, s := range states {
		r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
			State: s, Since: at}
		if i > 0 {
			r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
		}
		if s == link.StateUnhealthy {
			r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
		}
		h.Resources = append(h.Resources, r)
	}
	return h
}

func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	k := conditionsFrom
	const key = "module.letta.anchor.unhealthy"
	openKeys := func() []string {
		t.Helper()
		list, err := k.Open(ctx)
		if err != nil {
			t.Fatal(err)
		}
		var keys []string
		for _, c := range list {
			keys = append(keys, c.Key)
		}
		return keys
	}

	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
		t.Fatal(err)
	}
	// One statement: nothing raised, and `node show` lists it as unconfirmed.
	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
		h0.Add(time.Minute)); err != nil {
		t.Fatal(err)
	}
	if keys := openKeys(); len(keys) != 0 {
		t.Fatalf("one statement raised %v", keys)
	}
	kept, had, err := inv.HealthOf(ctx, "anchor")
	if err != nil || !had {
		t.Fatalf("the statement was not kept: %v %v", had, err)
	}
	if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
		!strings.Contains(lines, "letta.server") {
		t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
	}

	// The second in a row raises it — the module's own, never the other module's on the machine.
	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
		h0.Add(2*time.Minute)); err != nil {
		t.Fatal(err)
	}
	if keys := openKeys(); len(keys) != 1 || keys[0] != key {
		t.Fatalf("two statements raised %v, not %s", keys, key)
	}
	c, _, _ := k.Get(ctx, key)
	if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
		!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
		!strings.Contains(c.Evidence[0].Said, "letta-server") {
		t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
	}

	// Standing four hours, it is urgent.
	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
		time.Now().Add(5*time.Hour)); err != nil {
		t.Fatal(err)
	}
	if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
		t.Fatalf("unhealthy for four hours is still %s", c.Severity)
	}

	// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
		h0.Add(6*time.Hour)); err != nil {
		t.Fatal(err)
	}
	if keys := openKeys(); len(keys) != 0 {
		t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
	}
	// And the streak starts again: one unhealthy statement after it raises nothing.
	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
		t.Fatal(err)
	}
	if keys := openKeys(); len(keys) != 0 {
		t.Fatalf("one statement after a clearing raised %v", keys)
	}
}

func TestAnOlderHealthStatementIsRefused(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	before := healthRefused.Load()
	if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
		t.Fatal(err)
	}
	// An event said before the report that overtook it arrives late.
	if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
		t.Fatal(err)
	}
	kept, _, err := inv.HealthOf(ctx, "anchor")
	if err != nil {
		t.Fatal(err)
	}
	if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
		t.Fatalf("the older statement replaced the newer: %+v", kept)
	}
	if healthRefused.Load() != before+1 {
		t.Fatalf("the refusal was not counted")
	}
}

// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
	if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
		t.Fatal(err)
	}
	if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
		t.Fatalf("health kept for an engine that said none: %v %v", had, err)
	}
	if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
		t.Fatalf("node show: %v", lines)
	}
	// And one that does, through the report, is kept.
	h := aStatement(time.Now().UTC(), link.StateHealthy)
	if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
		t.Fatal(err)
	}
	if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
		t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
	}
}

// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	reads := func() bool {
		t.Helper()
		in, err := readComposeInput(ctx, open)
		if err != nil {
			t.Fatal(err)
		}
		node, _ := nodeOf(in.Input, "anchor")
		return node.Stated && node.Contract >= compose.ReadinessContract
	}
	if reads() {
		t.Fatal("an engine that never stated anything is sent health")
	}
	if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
		t.Fatal(err)
	}
	if reads() {
		t.Fatal("an engine judging liveness alone is sent health")
	}
	later := aStatement(h0.Add(time.Minute), link.StateHealthy)
	later.Contract = link.ReadinessContract
	if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
		t.Fatal(err)
	}
	if !reads() {
		t.Fatal("an engine that reads health is not sent it")
	}
}