Keep what each machine says of what it runs, raise it, and gate on it (hq ADR 0240, to-be 48 Phase A)

The gate judged a module by what the mesh saw from outside, so a container that
crash-looped after it applied passed it. Each machine's node-engine now states
the health of every long-running resource it runs; the controller keeps the
newest statement per machine, raises module.<module>.<machine>.unhealthy on the
second statement in a row, clears it on the first that does not say it, and the
gate passes a module only when every long-running resource of it is stated
healthy since the send. An engine that states nothing is judged as before.

internal/link/bus.go

@@ -75,8 +75,16 @@ const (
 	// ToolsAliveSubjects is every machine's node tools saying they are there (novox/hq to-be 45 §3,
 	// S11): core NATS like the host's, for the same reason.
 	ToolsAliveSubjects = "mesh.control.*.tools-alive"
+
+	// HealthSubjects is every machine's health statement between its reports (novox/hq ADR 0240): core
+	// NATS like the heartbeat, because a statement lost is said again within a minute while anything is
+	// not healthy, and the next report carries it whatever happens.
+	HealthSubjects = "mesh.control.*.health"
 )
 
+// HealthSubject is one machine's health statement.
+func HealthSubject(node string) string { return "mesh.control." + node + ".health" }
+
 // ReportSubject is where one node says what it did. On the CONTROL stream, because it is the
 // message the store-window guarantee is about (ADR 0083).
 func ReportSubject(node string) string { return "mesh.control." + node + ".report" }

internal/link/contracts.go

@@ -35,6 +35,10 @@ var Contracts = map[string]Contract{
 	KindHeartbeat: {Unordered: "a word that the machine is there: the newest heard is the newest said, and one " +
 		"lost is the next one"},
 	KindToolsHeartbeat: {Unordered: "a word that the node tools are there, as a machine's heartbeat"},
+	KindHealth: {Ordered: "by the time the node-engine looked (`at`, on the machine's clock): a statement older " +
+		"than the one kept for that machine is refused, so an event arriving after a newer report does not undo it " +
+		"(novox/hq ADR 0240)",
+		Tests: []string{"TestAnOlderHealthStatementIsRefused"}},
 	KindEnrolment: {Unordered: "a request answered once, under a token spent once: a second presentation is " +
 		"refused by the token, not by an order (issue 083)",
 		Tests: []string{"TestAnEnrolmentMetByAHeldTokenIsAskedToTryAgain"}},

internal/link/contracts_test.go

@@ -16,7 +16,7 @@ import (
 func TestEveryConsumedKindHasAContract(t *testing.T) {
 	// What the controller can be handed, from the subjects it is granted and the ones it derives.
 	subjects := []string{EnrolSubject, BuiltSubject, ReportSubject("anchor"), AliveSubject("anchor"),
-		ToolsAliveSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
+		ToolsAliveSubject("anchor"), HealthSubject("anchor"), "mesh.mod.postgres.event.provisioner.failing"}
 	subjects = append(subjects, broker.ControllerFollows...)
 	kinds := map[string]bool{}
 	for _, s := range subjects {

internal/link/health.go

@@ -0,0 +1,56 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+)
+
+// A machine's health statement (novox/hq ADR 0240, to-be 48 §4).
+//
+// **The node-engine owns every verdict; the controller keeps the last word per machine.** A statement
+// arrives in every report and as an event between reports — on each change, and again every minute while
+// a resource is not healthy. What the controller does with it — keep it, refuse an older one, raise
+// `module.<module>.<machine>.unhealthy` on the second statement that says so — is the Healths it is given.
+
+// Healths keeps what machines state of their long-running resources.
+type Healths interface {
+	// Stated keeps one machine's statement, from a report or the event. An older statement than the one
+	// kept is refused there, by its time on the machine.
+	Stated(ctx context.Context, node string, h Health) error
+}
+
+// Hears says where the machines' health statements are kept. Their subscription is the heartbeats':
+// core, and always made, so nothing is asked of the bus here.
+func (s *Server) Hears(h Healths) { s.healths = h }
+
+// healthSaid acts on one health event. Core NATS, so there is nothing to hold: a statement that could not
+// be kept is said in the log, and the next one — a minute away while anything is not healthy — is kept.
+func (s *Server) healthSaid(ctx context.Context, m Control) {
+	defer func() { _ = m.Took() }()
+	var said HealthSaid
+	if err := json.Unmarshal(m.Body(), &said); err != nil || said.Health.Contract == 0 {
+		return
+	}
+	// The machine is the one in the subject the bus let it publish on, never the body's.
+	node, ok := nodeOfHealth(m.Subject())
+	if !ok || s.healths == nil {
+		return
+	}
+	if err := s.healths.Stated(ctx, node, said.Health); err != nil {
+		s.log.Printf("could not keep what %s says of its resources' health: %v", node, err)
+	}
+}
+
+// nodeOfHealth is the machine a health statement names in its subject.
+func nodeOfHealth(subject string) (string, bool) {
+	rest, ok := strings.CutPrefix(subject, "mesh.control.")
+	if !ok {
+		return "", false
+	}
+	node, kind, ok := strings.Cut(rest, ".")
+	if !ok || kind != "health" || node == "" || strings.Contains(node, ".") {
+		return "", false
+	}
+	return node, true
+}

internal/link/health_test.go

@@ -0,0 +1,37 @@
+package link
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// A health event is kept as the machine its subject names says it, whatever its body claims, and taken
+// (novox/hq ADR 0240): core NATS, nothing to hold.
+
+type keptHealths struct{ by map[string]Health }
+
+func (k *keptHealths) Stated(_ context.Context, node string, h Health) error {
+	k.by[node] = h
+	return nil
+}
+
+func TestAHealthEventIsKeptAsTheMachineInItsSubject(t *testing.T) {
+	s, in := serving()
+	kept := &keptHealths{by: map[string]Health{}}
+	s.Hears(kept)
+	to := &settled{}
+	m := in.sends(t, to, KindHealth, HealthSaid{Node: "laptop", Health: Health{Contract: LivenessContract, At: time.Now(),
+		Resources: []ResourceHealth{{Module: "letta", Resource: "letta.server", State: StateUnhealthy}}}}).(*fakeControl)
+	m.subject = HealthSubject("anchor")
+	s.act(t.Context(), m)
+	if !to.acked {
+		t.Fatal("a health event was not taken")
+	}
+	if _, lied := kept.by["laptop"]; lied || len(kept.by["anchor"].Resources) != 1 {
+		t.Fatalf("kept %+v: the machine is the subject's, never the body's", kept.by)
+	}
+	if kind, ok := kindOfSubject(HealthSubject("anchor")); !ok || kind != KindHealth {
+		t.Fatalf("%s is not read as a health statement", HealthSubject("anchor"))
+	}
+}

internal/link/protocol.go

@@ -262,6 +262,13 @@ type Report struct {
 	// `witness` and `not-reversible` are sent only to a machine whose report carries it.
 	Witness int `json:"witness,omitempty"`
 
+	// Health is the machine's word on every long-running resource it runs for a module (novox/hq ADR
+	// 0240, to-be 48 §4; mesh-host's internal/liveness): its state, since when, its failing streak and
+	// the restarts its node-engine counted. **Absent from an engine older than the judging**, which is
+	// read as "not known" — never as healthy, never as a reason to raise anything; present with no
+	// resources from a machine that runs nothing long-lived.
+	Health *Health `json:"health,omitempty"`
+
 	// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
 	// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
 	// overlay key and tunnel and nothing else.
@@ -404,3 +411,45 @@ func EnrolProof(secret string, public []byte, overlay, sealing, serving string)
 	return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
 		"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
 }
+
+// LivenessContract is the version of the health statement this controller reads (ADR 0240 Phase A).
+const LivenessContract = 1
+
+// Health is one statement of a machine's long-running resources (to-be 48 §4): in every report, as the
+// event HealthSubject between reports on each change, and again every minute while one is not healthy.
+// The node-engine's own (mesh-host internal/link Health); a test on each side holds the field names.
+type Health struct {
+	Contract int `json:"contract"`
+	// At is when the engine looked, on the machine's clock: the order of its statements.
+	At        time.Time        `json:"at"`
+	Resources []ResourceHealth `json:"resources"`
+}
+
+// The states a resource is said in (ADR 0240 §4).
+const (
+	StateHealthy   = "healthy"
+	StateUnhealthy = "unhealthy"
+	StateStarting  = "starting"
+	StateHeld      = "held"
+	StateUnknown   = "unknown"
+)
+
+// ResourceHealth is one long-running resource's state.
+type ResourceHealth struct {
+	Module   string    `json:"module"`
+	Resource string    `json:"resource"`
+	Kind     string    `json:"kind"`
+	Target   string    `json:"target"`
+	State    string    `json:"state"`
+	Reason   string    `json:"reason,omitempty"`
+	Since    time.Time `json:"since"`
+	Streak   int       `json:"streak,omitempty"`
+	Restarts int       `json:"restarts,omitempty"`
+}
+
+// HealthSaid is the health event's body: the machine and its statement. The machine is read from the
+// subject the bus let it publish on, never from here.
+type HealthSaid struct {
+	Node   string `json:"node"`
+	Health Health `json:"health"`
+}

internal/link/protocol_test.go

@@ -25,6 +25,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
 			[]string{"protocol", "address", "port", "by", "published", "container-port"}},
 		{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
 			[]string{"node", "secret", "public_key"}},
+		// novox/hq ADR 0240: every long-running resource's health, in every report and in its own event.
+		{Report{Node: "n", Health: &Health{Contract: LivenessContract}}, []string{"node", "health"}},
+		{Health{Contract: LivenessContract, Resources: []ResourceHealth{}}, []string{"contract", "at", "resources"}},
+		{ResourceHealth{Module: "m", Resource: "m.r", Kind: "container", Target: "t", State: StateUnhealthy,
+			Reason: "restarting", Streak: 2, Restarts: 3},
+			[]string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts"}},
+		{HealthSaid{Node: "n"}, []string{"node", "health"}},
 	} {
 		raw, err := json.Marshal(c.value)
 		if err != nil {

internal/link/receive.go

@@ -30,8 +30,10 @@ const (
 	KindHeartbeat = "heartbeat"
 	// KindToolsHeartbeat is a machine's node tools saying they are there (novox/hq to-be 45 S11).
 	KindToolsHeartbeat = "tools-heartbeat"
-	KindBuilt          = "built"
-	KindModuleMoved    = "module-moved"
+	// KindHealth is a machine's statement of its long-running resources' health (novox/hq ADR 0240).
+	KindHealth      = "health"
+	KindBuilt       = "built"
+	KindModuleMoved = "module-moved"
 	// KindSourceMoved is the forge announcing a merge: a source moved, and what it produces is
 	// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
 	KindSourceMoved = "source-moved"

internal/link/receive_nats.go

@@ -108,6 +108,13 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
 		return fmt.Errorf("subscribing to the node tools' heartbeats: %w", err)
 	}
 	defer func() { _ = toolsAlive.Unsubscribe() }()
+	// And what each machine says of its long-running resources between reports (novox/hq ADR 0240): core
+	// too, and said again while it matters.
+	health, err := conn.ChanSubscribe(HealthSubjects, beats)
+	if err != nil {
+		return fmt.Errorf("subscribing to the machines' health: %w", err)
+	}
+	defer func() { _ = health.Unsubscribe() }()
 
 	// The events the controller follows, when something is listening for them.
 	var events chan *nats.Msg
@@ -239,6 +246,8 @@ func kindOfSubject(subject string) (string, bool) {
 			return KindHeartbeat, true
 		case "tools-alive":
 			return KindToolsHeartbeat, true
+		case "health":
+			return KindHealth, true
 		}
 	}
 	switch subject {

internal/link/serve.go

@@ -94,6 +94,8 @@ type Server struct {
 	retirements Retirements
 	// checker asks for a pull request's merge check (novox/hq to-be 45 §9).
 	checker Checker
+	// healths keeps what machines say of their long-running resources (novox/hq ADR 0240).
+	healths Healths
 
 	log *log.Logger
 	// giveUp is how long one message is held for the store; zero means GiveUpAfter.
@@ -194,6 +196,8 @@ func (s *Server) act(ctx context.Context, m Control) {
 		s.heartbeat(m)
 	case KindToolsHeartbeat:
 		s.toolsHeartbeat(m)
+	case KindHealth:
+		s.healthSaid(ctx, m)
 	case KindBuilt:
 		s.wasBuilt(ctx, m)
 	case KindModuleMoved: