Keep what each machine says of what it runs, raise it, and gate on it (hq ADR 0240, to-be 48 Phase A)

The gate judged a module by what the mesh saw from outside, so a container that
crash-looped after it applied passed it. Each machine's node-engine now states
the health of every long-running resource it runs; the controller keeps the
newest statement per machine, raises module.<module>.<machine>.unhealthy on the
second statement in a row, clears it on the first that does not say it, and the
gate passes a module only when every long-running resource of it is stated
healthy since the send. An engine that states nothing is judged as before.

cmd/mesh-controller/gate_test.go

@@ -346,6 +346,48 @@ func TestTheHealthDefinitions(t *testing.T) {
 		!strings.Contains(why, "first run") {
 		t.Errorf("a controller not ready is %v (%s)", h, why)
 	}
+	// What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running
+	// resource of it on that machine is stated healthy since the send; starting and unhealthy wait.
+	f = applied("anchor")
+	stated := func(state, reason string, heard time.Time) {
+		f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard,
+			Resources: []inventory.ResourceHealth{
+				{Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason},
+				{Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}}
+	}
+	plain := catalogue.Manifest{Module: "app"}
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
+		t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why)
+	}
+	stated(link.StateStarting, "", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") {
+		t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why)
+	}
+	stated(link.StateUnhealthy, "restarting", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") {
+		t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why)
+	}
+	stated(link.StateHealthy, "", since.Add(-time.Minute))
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
+		t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why)
+	}
+	stated(link.StateHealthy, "", now)
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood {
+		t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why)
+	}
+	f.healthErr = errors.New("the store is away")
+	if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet {
+		t.Errorf("health that cannot be read is never read as healthy: %v", h)
+	}
+	// And the condition it raises after the send holds it, as every condition about it does.
+	f.healthErr = nil
+	f.judged = true
+	f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{
+		Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}}
+	if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") {
+		t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why)
+	}
+
 	// A machine that refused what it was sent: broken.
 	f = applied("anchor")
 	r := f.reports["anchor"]