Keep what each machine says of what it runs, raise it, and gate on it (hq ADR 0240, to-be 48 Phase A)

The gate judged a module by what the mesh saw from outside, so a container that
crash-looped after it applied passed it. Each machine's node-engine now states
the health of every long-running resource it runs; the controller keeps the
newest statement per machine, raises module.<module>.<machine>.unhealthy on the
second statement in a row, clears it on the first that does not say it, and the
gate passes a module only when every long-running resource of it is stated
healthy since the send. An engine that states nothing is judged as before.

cmd/mesh-controller/gate.go

@@ -105,6 +105,10 @@ type gateFacts struct {
 	// holder is who holds the controller lease, for judging the controller.
 	holder    *lease.Holder
 	holderErr error
+	// health is each machine's newest health statement (ADR 0240); a machine absent never stated one.
+	// healthErr is why they could not be read.
+	health    map[string]inventory.NodeHealth
+	healthErr error
 }
 
 // gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
@@ -127,6 +131,9 @@ var gatherGateFacts = func(ctx context.Context, open *stores, component string)
 	for _, n := range nodes {
 		f.engines[n.Name] = n.HostVersion
 	}
+	// What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as
+	// healthy.
+	f.health, f.healthErr = inv.Healths(ctx)
 	if d := doctorFrom; d != nil {
 		if d.keeper != nil {
 			f.judged = true
@@ -241,6 +248,11 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
 				return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module)
 			}
 		}
+		// **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that
+		// machine, in a statement heard since the send. A resource still starting makes the judging wait.
+		if h, why := moduleHealthWord(module, machine, since, f); h != healthGood {
+			return h, why
+		}
 	}
 	return healthGood, ""
 }

cmd/mesh-controller/gate_test.go

@@ -346,6 +346,48 @@ func TestTheHealthDefinitions(t *testing.T) {
 		!strings.Contains(why, "first run") {
 		t.Errorf("a controller not ready is %v (%s)", h, why)
 	}
+	// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
+	// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
+	f = applied("anchor")
+	stated := func(state, reason string, heard time.Time) {
+		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
+			Resources: []inventory.ResourceHealth{
+				{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
+				{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
+	}
+	plain := catalogue.Manifest{Module: "app"}
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
+		t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
+	}
+	stated(link.StateStarting, "", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
+		t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
+	}
+	stated(link.StateUnhealthy, "restarting", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
+		t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
+	}
+	stated(link.StateHealthy, "", since.Add(-time.Minute))
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
+		t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
+	}
+	stated(link.StateHealthy, "", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
+		t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
+	}
+	f.healthErr = errors.New("the store is away")
+	if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
+		t.Errorf("health that cannot be read is never read as healthy: %v", h)
+	}
+	// And the condition it raises after the send holds it, as every condition about it does.
+	f.healthErr = nil
+	f.judged = true
+	f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
+		Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
+		t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
+	}
+
 	// A machine that refused what it was sent: broken.
 	f = applied("anchor")
 	r := f.reports["anchor"]

cmd/mesh-controller/module_health.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A module says how it is healthy, and the node-engine judges it (novox/hq ADR 0240, to-be 48 §4 and §5,
+// Phase A).
+//
+// **The node-engine owns every verdict; the controller keeps the last word and raises the condition.** A
+// machine states, in every report and as an event between reports, the state of every long-running
+// resource it runs for a module. The controller keeps the newest statement per machine (node_health), and
+// raises `module.<module>.<machine>.unhealthy` when two statements in a row say a resource of the module
+// is unhealthy — one statement is listed as unconfirmed, as the self-check does a finding one look can be
+// wrong about (to-be 45 §4, issue 277) — and clears it on the first that does not. The release gate reads
+// the stated health: a judging passes a module only when every long-running resource of it on that
+// machine is stated healthy, so a resource still starting is not yet a pass.
+//
+// **An engine older than the judging states nothing**, and its machine's health is not known: never
+// healthy, never a reason to raise anything, and the gate judges it as it did before.
+
+// The condition a module's health raises.
+const (
+	kindModuleUnhealthy = "module-unhealthy"
+	// sourceHealth is what raised it: the machine's own statement.
+	sourceHealth = "health"
+	// moduleUnhealthyUrgentAfter is how long it stands before it is urgent (to-be 48 §4).
+	moduleUnhealthyUrgentAfter = 4 * time.Hour
+	// moduleUnhealthyAfter is how many statements in a row raise it.
+	moduleUnhealthyAfter = 2
+)
+
+// healthRefused counts the statements refused as older than the one kept, for the log and a test.
+var healthRefused atomic.Int64
+
+// moduleHealth keeps what the machines state, for the link (link.Healths).
+type moduleHealth struct {
+	inv    *inventory.Inventory
+	keeper func() *conditions.Keeper
+}
+
+func (m moduleHealth) Stated(ctx context.Context, node string, h link.Health) error {
+	return stateHealth(ctx, m.inv, m.keeper(), node, h, time.Now())
+}
+
+// stateHealth keeps one machine's statement and raises or clears its modules' conditions from it. An
+// older statement than the one kept is refused, by when the engine looked.
+func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Keeper, node string, h link.Health,
+	now time.Time) error {
+	if h.Contract == 0 {
+		return nil
+	}
+	prev, had, err := inv.HealthOf(ctx, node)
+	if err != nil {
+		return err
+	}
+	if had && h.At.Before(prev.SaidAt) {
+		healthRefused.Add(1)
+		return nil
+	}
+	unhealthy := map[string][]inventory.ResourceHealth{}
+	resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
+	for _, r := range h.Resources {
+		kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
+			State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts}
+		resources = append(resources, kept)
+		if r.State == link.StateUnhealthy && r.Module != "" {
+			unhealthy[r.Module] = append(unhealthy[r.Module], kept)
+		}
+	}
+	streaks := map[string]int{}
+	for module := range unhealthy {
+		streaks[module] = prev.Streaks[module] + 1
+	}
+	stored, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: h.Contract, SaidAt: h.At,
+		HeardAt: now, Resources: resources, Streaks: streaks})
+	if err != nil || !stored {
+		if err == nil {
+			healthRefused.Add(1)
+		}
+		return err
+	}
+	if k == nil {
+		return nil
+	}
+	return judgeModuleHealth(ctx, k, node, unhealthy, streaks, now)
+}
+
+// judgeModuleHealth raises a module's condition on a machine on the second statement in a row that says a
+// resource of it is unhealthy — or on the first while it is already open — and clears every one this
+// statement no longer says.
+func judgeModuleHealth(ctx context.Context, k *conditions.Keeper, node string,
+	unhealthy map[string][]inventory.ResourceHealth, streaks map[string]int, now time.Time) error {
+	open, err := k.Open(ctx)
+	if err != nil {
+		return err
+	}
+	standing := map[string]conditions.Condition{}
+	for _, c := range open {
+		if c.Kind == kindModuleUnhealthy && c.Subject.Machine == node {
+			standing[c.Key] = c
+		}
+	}
+	var problems []string
+	modules := make([]string, 0, len(unhealthy))
+	for m := range unhealthy {
+		modules = append(modules, m)
+	}
+	sort.Strings(modules)
+	seen := map[string]bool{}
+	for _, m := range modules {
+		o := moduleUnhealthyObservation(m, node, unhealthy[m])
+		seen[o.Key()] = true
+		c, isOpen := standing[o.Key()]
+		if streaks[m] < moduleUnhealthyAfter && !isOpen {
+			continue // unconfirmed: one statement can be wrong; `node show` lists it
+		}
+		if isOpen && now.Sub(c.Raised) >= moduleUnhealthyUrgentAfter {
+			o.Severity = conditions.Urgent
+		}
+		if _, err := k.Observe(ctx, o); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	for key, c := range standing {
+		if seen[key] {
+			continue
+		}
+		module := strings.TrimSuffix(c.Subject.ID, "."+node)
+		if _, err := k.Clear(ctx, key, fmt.Sprintf("%s says no resource of %s is unhealthy", node, module)); err != nil {
+			problems = append(problems, err.Error())
+		}
+	}
+	if len(problems) > 0 {
+		return fmt.Errorf("%s", strings.Join(problems, "; "))
+	}
+	return nil
+}
+
+// moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module,
+// the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence.
+func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation {
+	var words, said []string
+	for _, r := range rs {
+		words = append(words, fmt.Sprintf("its %s %s %s", r.Kind, r.Resource, reasonWords(r)))
+		said = append(said, fmt.Sprintf("%s (%s %s): %s, %d look(s) in a row, %d restart(s) counted, since %s",
+			r.Resource, r.Kind, r.Target, orNotSaid(r.Reason), r.Streak, r.Restarts,
+			r.Since.UTC().Format("2006-01-02 15:04:05 MST")))
+	}
+	return conditions.Observation{Scope: conditions.ScopeModule, ID: module + "." + node, Token: "unhealthy",
+		Kind: kindModuleUnhealthy, Machine: node, Severity: conditions.Warning, Source: sourceHealth,
+		Summary: fmt.Sprintf("%s on %s is not healthy: %s", module, node, strings.Join(words, "; ")),
+		Said:    strings.Join(said, "; ")}
+}
+
+// reasonWords is why a resource is unhealthy, as a person reads it.
+func reasonWords(r inventory.ResourceHealth) string {
+	switch r.Reason {
+	case "restarting":
+		return fmt.Sprintf("keeps restarting (%d restart(s) counted)", r.Restarts)
+	case "down":
+		return "is not running"
+	case "":
+		return "is unhealthy"
+	}
+	return "is unhealthy: " + r.Reason
+}
+
+func orNotSaid(s string) string {
+	if s == "" {
+		return "no reason said"
+	}
+	return s
+}
+
+// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236
+// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since
+// the send; not yet otherwise, saying which. A machine that never stated health is judged as before.
+func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) {
+	if f.healthErr != nil {
+		return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error())
+	}
+	h, states := f.health[machine]
+	if !states {
+		return healthGood, ""
+	}
+	if h.HeardAt.Before(since) {
+		return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
+	}
+	for _, r := range h.Resources {
+		if r.Module != module {
+			continue
+		}
+		switch r.State {
+		case link.StateHealthy:
+		case link.StateStarting:
+			return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine)
+		case link.StateUnhealthy:
+			return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r))
+		default:
+			return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State,
+				reasonAfter(r.Reason))
+		}
+	}
+	return healthGood, ""
+}
+
+func reasonAfter(s string) string {
+	if s == "" {
+		return ""
+	}
+	return ": " + s
+}
+
+// healthLines is what `node show` says of a machine's long-running resources: each with its state and
+// since when, an unhealthy one said once marked unconfirmed.
+func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
+	if !had {
+		return []string{"  its node-engine does not say how what it runs is — it is older than the judging (ADR 0240)"}
+	}
+	if len(h.Resources) == 0 {
+		return []string{fmt.Sprintf("  it runs nothing long-lived for a module (said %s ago)", roughly(now.Sub(h.HeardAt)))}
+	}
+	out := []string{fmt.Sprintf("  what it runs, as it said %s ago:", roughly(now.Sub(h.HeardAt)))}
+	for _, r := range h.Resources {
+		line := fmt.Sprintf("    %-10s %-34s %s %s, since %s", r.State, r.Resource, r.Kind, r.Target,
+			r.Since.Local().Format("2006-01-02 15:04"))
+		if r.Reason != "" {
+			line += " — " + r.Reason
+		}
+		if r.Restarts > 0 {
+			line += fmt.Sprintf(", %d restart(s) counted", r.Restarts)
+		}
+		if r.State == link.StateUnhealthy && h.Streaks[r.Module] < moduleUnhealthyAfter {
+			line += " (unconfirmed: said once)"
+		}
+		out = append(out, line)
+	}
+	return out
+}

cmd/mesh-controller/module_health_test.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// A module's stated health, as the controller keeps it and raises from it (novox/hq ADR 0240, "how it is
+// checked", rule 4): one unhealthy statement raises nothing and is listed unconfirmed; two raise; a
+// healthy one clears; a condition from before the send clears at the new build's start; an older
+// statement is refused; and an engine that states nothing raises nothing.
+
+var h0 = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
+
+func aStatement(at time.Time, states ...string) link.Health {
+	h := link.Health{Contract: link.LivenessContract, At: at}
+	for i, s := range states {
+		r := link.ResourceHealth{Module: "letta", Resource: "letta.server", Kind: "container", Target: "letta-server",
+			State: s, Since: at}
+		if i > 0 {
+			r.Module, r.Resource, r.Target = "mqtt", "mqtt.broker", "mosquitto.service"
+		}
+		if s == link.StateUnhealthy {
+			r.Reason, r.Restarts, r.Streak = "restarting", 4, 2
+		}
+		h.Resources = append(h.Resources, r)
+	}
+	return h
+}
+
+func TestTwoUnhealthyStatementsRaiseTheModulesConditionAndAHealthyOneClearsIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	k := conditionsFrom
+	const key = "module.letta.anchor.unhealthy"
+	openKeys := func() []string {
+		t.Helper()
+		list, err := k.Open(ctx)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var keys []string
+		for _, c := range list {
+			keys = append(keys, c.Key)
+		}
+		return keys
+	}
+
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0, link.StateHealthy, link.StateHealthy), h0); err != nil {
+		t.Fatal(err)
+	}
+	// One statement: nothing raised, and `node show` lists it as unconfirmed.
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(time.Minute), link.StateUnhealthy, link.StateHealthy),
+		h0.Add(time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	if keys := openKeys(); len(keys) != 0 {
+		t.Fatalf("one statement raised %v", keys)
+	}
+	kept, had, err := inv.HealthOf(ctx, "anchor")
+	if err != nil || !had {
+		t.Fatalf("the statement was not kept: %v %v", had, err)
+	}
+	if lines := strings.Join(healthLines(kept, had, h0.Add(time.Minute)), "\n"); !strings.Contains(lines, "unconfirmed") ||
+		!strings.Contains(lines, "letta.server") {
+		t.Fatalf("node show does not list the first statement as unconfirmed:\n%s", lines)
+	}
+
+	// The second in a row raises it — the module's own, never the other module's on the machine.
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(2*time.Minute), link.StateUnhealthy, link.StateHealthy),
+		h0.Add(2*time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	if keys := openKeys(); len(keys) != 1 || keys[0] != key {
+		t.Fatalf("two statements raised %v, not %s", keys, key)
+	}
+	c, _, _ := k.Get(ctx, key)
+	if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverSelf || c.Subject.Machine != "anchor" ||
+		!strings.Contains(c.Summary, "letta on anchor") || !strings.Contains(c.Summary, "keeps restarting") ||
+		!strings.Contains(c.Evidence[0].Said, "letta-server") {
+		t.Fatalf("the condition does not say it in words with its evidence: %+v", c)
+	}
+
+	// Standing four hours, it is urgent.
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(5*time.Hour), link.StateUnhealthy, link.StateHealthy),
+		time.Now().Add(5*time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+	if c, _, _ := k.Get(ctx, key); c.Severity != conditions.Urgent {
+		t.Fatalf("unhealthy for four hours is still %s", c.Severity)
+	}
+
+	// A new build's start — every start begins in `starting` — clears it: what follows is the new build's.
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(6*time.Hour), link.StateStarting, link.StateHealthy),
+		h0.Add(6*time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+	if keys := openKeys(); len(keys) != 0 {
+		t.Fatalf("the first statement that says no resource is unhealthy did not clear it: %v", keys)
+	}
+	// And the streak starts again: one unhealthy statement after it raises nothing.
+	if err := stateHealth(ctx, inv, k, "anchor", aStatement(h0.Add(7*time.Hour), link.StateUnhealthy), h0.Add(7*time.Hour)); err != nil {
+		t.Fatal(err)
+	}
+	if keys := openKeys(); len(keys) != 0 {
+		t.Fatalf("one statement after a clearing raised %v", keys)
+	}
+}
+
+func TestAnOlderHealthStatementIsRefused(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	before := healthRefused.Load()
+	if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0.Add(time.Minute), link.StateHealthy), h0); err != nil {
+		t.Fatal(err)
+	}
+	// An event said before the report that overtook it arrives late.
+	if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateUnhealthy), h0); err != nil {
+		t.Fatal(err)
+	}
+	kept, _, err := inv.HealthOf(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !kept.SaidAt.Equal(h0.Add(time.Minute)) || kept.Resources[0].State != link.StateHealthy {
+		t.Fatalf("the older statement replaced the newer: %+v", kept)
+	}
+	if healthRefused.Load() != before+1 {
+		t.Fatalf("the refusal was not counted")
+	}
+}
+
+// **An engine older than the judging states nothing**: its reports raise nothing, keep nothing, and the
+// gate judges its machine as before — never healthy for having said nothing, never unhealthy.
+func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	l := nudgingListener{Enrolment: link.Enrolment{Inventory: open.inventory}}
+	if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Applied: []string{"letta.server"}}); err != nil {
+		t.Fatal(err)
+	}
+	if _, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || had {
+		t.Fatalf("health kept for an engine that said none: %v %v", had, err)
+	}
+	if lines := healthLines(inventory.NodeHealth{}, false, time.Now()); !strings.Contains(lines[0], "older than the judging") {
+		t.Fatalf("node show: %v", lines)
+	}
+	// And one that does, through the report, is kept.
+	h := aStatement(time.Now().UTC(), link.StateHealthy)
+	if _, err := l.Heard(ctx, link.Report{Node: "anchor", Declared: "d1", Health: &h}); err != nil {
+		t.Fatal(err)
+	}
+	if kept, had, err := open.inventory.HealthOf(ctx, "anchor"); err != nil || !had || len(kept.Resources) != 1 {
+		t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
+	}
+}

cmd/mesh-controller/nodes.go

@@ -537,6 +537,17 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
 		}
 	}
 
+	// What it runs for its modules, and how each is (novox/hq ADR 0240): "is it working" answered without
+	// a terminal on the machine.
+	if h, had, err := inv.HealthOf(ctx, name); err != nil {
+		fmt.Printf("\n  what it says of what it runs could NOT be read: %v\n", err)
+	} else {
+		fmt.Println()
+		for _, line := range healthLines(h, had, time.Now()) {
+			fmt.Println(line)
+		}
+	}
+
 	held, err := inv.Profile(ctx, name)
 	if err != nil {
 		return err

cmd/mesh-controller/push.go

@@ -182,6 +182,9 @@ func serve(ctx context.Context) (err error) {
 	if err := server.Watches(standings{keeper: func() *conditions.Keeper { return conditionsFrom }}); err != nil {
 		return err
 	}
+	// And what each machine says of what it runs between its reports, kept and raised from (novox/hq ADR
+	// 0240): the gate, `node show` and the conditions read it.
+	server.Hears(moduleHealth{inv: inv, keeper: func() *conditions.Keeper { return conditionsFrom }})
 	// And what they say about consumers the mesh stopped asking for: one waiting for a person is an
 	// urgent condition, and an act asked of a provider some other way is recorded by hand (ADR 0230).
 	if err := server.KeepsRetirements(retirements{keeper: func() *conditions.Keeper { return conditionsFrom },

cmd/mesh-controller/replays_test.go

@@ -1,9 +1,17 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"slices"
 	"testing"
+	"time"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
 )
 
 // The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what
@@ -111,3 +119,170 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
 		t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network)
 	}
 }
+
+// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq
+// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the
+// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the
+// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container
+// exits at start never passes its gate on the first machine, is put back there at the bound, and never
+// reaches the second.
+//
+// The machine is heard as a node-engine says it: its report of the apply, then the same account said again
+// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the
+// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the
+// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of
+// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
+// written only with what the controller had before the judging, for the prover to lay over that commit.
+func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+
+	crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
+		`"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` +
+		`"streak":5,"restarts":2}]}`)
+	if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("the engine's statement of the crash loop: %v", err)
+		}
+		crashLoop = raw
+	}
+	// `null` is an engine that states nothing — older than the judging — and its reports carry no health.
+	var stated map[string]any
+	if err := json.Unmarshal(crashLoop, &stated); err != nil {
+		t.Fatal(err)
+	}
+
+	for _, b := range []inventory.Build{
+		{ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app",
+			Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)},
+		{ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app",
+			Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)},
+	} {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit})
+		b.Manifest = manifest
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+	}
+	registerAt := func(commit string, asked time.Time) {
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit},
+			inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit,
+				Head: commit, Asked: asked}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	registerAt("c1", time.Now().Add(-2*time.Hour))
+	for _, n := range []string{"anchor", "laptop"} {
+		if _, err := inv.Assign(ctx, n, "app"); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	registerAt("c2", time.Now().Add(-time.Minute))
+
+	// The machine: what it is sent is applied, and its report says what runs is starting.
+	listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
+	sequence := int64(0)
+	say := func(node, digest, state string) {
+		t.Helper()
+		sequence++
+		health := map[string]any{}
+		for k, v := range stated {
+			health[k] = v
+		}
+		health["at"] = time.Now().UTC().Format(time.RFC3339Nano)
+		if state != "" && stated != nil {
+			resources := []any{}
+			for _, r := range stated["resources"].([]any) {
+				kept := map[string]any{}
+				for k, v := range r.(map[string]any) {
+					kept[k] = v
+				}
+				kept["state"], kept["reason"] = state, ""
+				resources = append(resources, kept)
+			}
+			health["resources"] = resources
+		}
+		said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest,
+			"report_sequence": sequence}
+		if stated != nil {
+			said["health"] = health
+		}
+		body, _ := json.Marshal(said)
+		var report link.Report
+		if err := json.Unmarshal(body, &report); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := listener.Heard(ctx, report); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var sent [][]string
+	last := map[string]string{}
+	n := 0
+	wasSend := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		sent = append(sent, append([]string(nil), names...))
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range names {
+			n++
+			digest := fmt.Sprintf("d-%s-%d", node, n)
+			if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil {
+				return nil, err
+			}
+			last[node] = digest
+			say(node, digest, "starting")
+		}
+		return names, nil
+	}
+	t.Cleanup(func() { sendRollout = wasSend })
+	wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
+	gateSettle, gateEvery = 0, 0
+	t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
+
+	built := time.Now().UTC()
+	plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
+		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
+		Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}}
+	if err := inv.SavePlan(ctx, &plan); err != nil {
+		t.Fatal(err)
+	}
+
+	advancePlans(ctx, open) // the first machine is sent the new build, and starts it
+	if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) {
+		t.Fatalf("sent %v, not the first machine first", sent)
+	}
+	advancePlans(ctx, open) // judged while it starts
+	// Its container exits at start, and the runtime restarts it: the machine says so, again and again.
+	for i := 0; i < 6 && len(sent) == 1; i++ {
+		say("anchor", last["anchor"], "")
+		advancePlans(ctx, open)
+	}
+	gateBound = -time.Second // and the bound passes
+	advancePlans(ctx, open)
+
+	p, err := inv.PlanByID(ctx, "plan-crashloop")
+	if err != nil {
+		t.Fatal(err)
+	}
+	gate := p.Modules["app"].Gate
+	for _, names := range sent {
+		if slices.Contains(names, "laptop") {
+			t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate)
+		}
+	}
+	if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed {
+		t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v",
+			p.State, p.Note, gate)
+	}
+	if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
+		t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
+	}
+}

cmd/mesh-controller/status_summary.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -204,6 +205,15 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
 	if news {
 		l.summary.nudge()
 	}
+	// What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions
+	// raised or cleared from it. Only from an account of the machine the store took.
+	if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" &&
+		l.Enrolment.Inventory != nil {
+		if herr := stateHealth(ctx, l.Enrolment.Inventory, conditionsFrom, report.Node, *report.Health, now); herr != nil {
+			fmt.Fprintf(os.Stderr, "mesh-controller: could not keep what %s says of its resources' health: %v\n",
+				report.Node, herr)
+		}
+	}
 	if err == nil && l.open != nil && startedWell(report) {
 		// Off the report's path: replacing a given value sends the machine, and a report waits for
 		// nothing it caused (novox/hq ADR 0228).