Judge whether what a module runs stays up, and say it (hq ADR 0240, to-be 48 Phase A)

A container that crash-looped after its compose applied passed every check the
gate had: nothing looked at what a module runs. The node-engine now judges every
long-running resource on every look — one read of the runtime, one per service
manager — keeps the restarts it counts across recreates and its own restarts,
and says the state in every report and as an event on change, again every minute
while not healthy. It reads only; nothing is restarted for being unhealthy.

cmd/mesh-host/main.go

@@ -34,6 +34,7 @@ import (
 	"github.com/novox/mesh-host/internal/identity"
 	"github.com/novox/mesh-host/internal/inventory"
 	"github.com/novox/mesh-host/internal/link"
+	"github.com/novox/mesh-host/internal/liveness"
 	"github.com/novox/mesh-host/internal/outward"
 	"github.com/novox/mesh-host/internal/profile"
 	"github.com/novox/mesh-host/internal/reachable"
@@ -1052,6 +1053,26 @@ func runLink(ctx context.Context, opts options) error {
 	sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
 	go sched.Run(ctx)
 
+	// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
+	// report this host makes carries what it says.
+	if j, err := liveness.Open(filepath.Join(filepath.Dir(opts.state), liveness.FileName),
+		&liveness.Exec{Run: apply.ExecRunner}); j != nil {
+		if err != nil {
+			say(err.Error())
+		}
+		windows := apply.WindowsIn(filepath.Dir(opts.state))
+		j.HeldNow = func(now time.Time) map[string]bool {
+			held := map[string]bool{}
+			for _, w := range windows.Open(now) {
+				for _, name := range w.Holds {
+					held[name] = true
+				}
+			}
+			return held
+		}
+		judging = j
+	}
+
 	// **Standing aside for a successor happens between reconciles and nowhere else** (novox/hq ADR
 	// 0141). A host that stood aside mid-apply is the half-configured machine this host exists to
 	// prevent, so the question is asked after an apply has finished and the answer is a clean exit —
@@ -1140,6 +1161,11 @@ func runLink(ctx context.Context, opts options) error {
 		queue.ReconcileDue()
 	}
 	go holdTheMachine(ctx, queue)
+	// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
+	// ADR 0240) — between applies, which is when a container crash-loops.
+	if judging != nil {
+		go judgeWhatRuns(aside, judging, queue, say)
+	}
 	// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
 	go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
 
@@ -1326,6 +1352,64 @@ func rousedBySignal(ctx context.Context) link.Roused {
 // changed it, and then for ever.
 const ReconcileEvery = 5 * time.Minute
 
+// judging is the serving host's liveness judge (novox/hq ADR 0240); nil in a one-shot command, which
+// reports no health, and in a test.
+var judging *liveness.Judge
+
+// How often a statement of health is said between reports: again every minute while anything is not
+// healthy (to-be 48 §4), so a lost event is not a lost fault; and every five minutes anyway, so a
+// controller that restarted knows a healthy machine's state without waiting for its next apply.
+const (
+	sayUnhealthyAgain = time.Minute
+	sayAnyway         = 5 * time.Minute
+)
+
+// judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
+// the bus when a state changed, again every minute while one is not healthy, and every five minutes
+// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
+// rule 6).
+func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
+	ticker := time.NewTicker(liveness.LookEvery)
+	defer ticker.Stop()
+	var lastSaid time.Time
+	owed := true
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+		st, changed := j.Look(ctx)
+		owed = owed || changed
+		since := time.Since(lastSaid)
+		if !owed && !(!st.Healthy() && since >= sayUnhealthyAgain) && since < sayAnyway {
+			continue
+		}
+		if changed {
+			for _, r := range st.Resources {
+				if r.State == liveness.Unhealthy {
+					say(fmt.Sprintf("%s (%s %s) is unhealthy: %s, %d restart(s) counted", r.ID, r.Kind, r.Target,
+						r.Reason, r.Restarts))
+				}
+			}
+		}
+		if queue.SayHealth(ctx, *healthAsReported(st)) {
+			lastSaid, owed = time.Now(), false
+		}
+	}
+}
+
+// healthAsReported is a statement as the report and the event carry it.
+func healthAsReported(st liveness.Statement) *link.Health {
+	h := &link.Health{Contract: link.LivenessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
+	for _, r := range st.Resources {
+		h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
+			Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
+			Restarts: r.Restarts})
+	}
+	return h
+}
+
 // holdTheMachine asks for a reconcile every ReconcileEvery. **It asks; it does not apply** (novox/hq
 // to-be 45 §6): the queue's worker does, when its turn comes, and a reconcile due while a delivery is
 // waiting is that delivery's apply.
@@ -1541,17 +1625,24 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
 	// is armed, one whose image, environment or cadence changed is re-armed, and one no longer
 	// declared is forgotten — and after a host restart the first apply rebuilds them all. A nil
 	// scheduler is the one-shot CLI path, which exits rather than staying up to fire anything.
+	held := map[string]bool{}
+	for _, h := range updated.Held {
+		held[h.ID] = true
+	}
 	if sched != nil {
-		held := map[string]bool{}
-		for _, h := range updated.Held {
-			held[h.ID] = true
-		}
 		sched.Sync(declared, held)
 	}
 
 	report := link.Report{Carried: carriedPorts(updated), Declared: digestOf(raw), Order: order, Host: runningVersion(),
 		Rollbacks: standingRollbacks(opts.state), Witness: link.WitnessContract,
 		Profile: profileAsReported(profile.Detect(ctx, profile.Default(nil), opts.timeout))}
+	// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
+	// applied, so every resource this apply started says `starting` and the gate waits for it.
+	if j := judging; j != nil {
+		j.Set(liveness.LongRunning(declared, held))
+		st, _ := j.Look(ctx)
+		report.Health = healthAsReported(st)
+	}
 	// Which of this machine's links face outside, for the filter the mesh writes around them
 	// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
 	// and an adopted one becomes converged without a further round trip. A machine that cannot read