Judge how a module says it is ready, beside whether it stays up (hq ADR 0240, to-be 48 Phase B)

Liveness alone could not see a web application whose port was open and whose
program ran while every request hung for eleven hours (issue 145). A resource
now carries the `health` its module declared: the engine makes http and tcp
looks itself from the machine to the endpoint's published port, reads a unit's
readiness from the show it already makes, hands an exec command or the image's
own check to the runtime as the container's check with the declared timing and
reads its state from the inspect it already makes, and asks a module's tool on
its own node tools. Starting until the check passed, unhealthy once its looks
after the grace fail the declared number of times; never more looks than the
measured budget; nothing restarted. The statement says contract 2, which tells
the controller this engine may be sent the field.

internal/liveness/runtime.go

@@ -2,6 +2,7 @@ package liveness
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
@@ -52,8 +53,11 @@ func (e *Exec) runtime(ctx context.Context) (string, error) {
 }
 
 // inspectFormat is what one inspect says of each container: its name, id, status, the runtime's restart
-// count and when its current run started.
-const inspectFormat = "{{.Name}}\t{{.Id}}\t{{.State.Status}}\t{{.RestartCount}}\t{{.State.StartedAt}}"
+// count, when its current run started — and the state of the check the runtime runs as the container's
+// own, read out of the whole state as one line of JSON, for a check declared exec or runtime (Phase B).
+// The whole state and not its Health: a container that carries no check has no such key, and the runtime
+// refuses a template naming a key that is not there — for every container in the read.
+const inspectFormat = "{{.Name}}\t{{.Id}}\t{{.State.Status}}\t{{.RestartCount}}\t{{.State.StartedAt}}\t{{json .State}}"
 
 // Containers is one inspect of every container named. A name the runtime does not have is not found;
 // a runtime that does not answer is an error — unknown, never down.
@@ -83,12 +87,47 @@ func (e *Exec) Containers(ctx context.Context, names []string) (map[string]Obser
 		name := strings.TrimPrefix(strings.TrimSpace(parts[0]), "/")
 		restarts, _ := strconv.ParseInt(strings.TrimSpace(parts[3]), 10, 64)
 		status := strings.TrimSpace(parts[2])
-		out[name] = Observed{Found: true, Identity: strings.TrimSpace(parts[1]), Running: status == "running",
+		o := Observed{Found: true, Identity: strings.TrimSpace(parts[1]), Running: status == "running",
 			Restarting: status == "restarting", Restarts: restarts, Started: strings.TrimSpace(parts[4])}
+		if len(parts) > 5 {
+			o.Health, o.HealthSaid = runtimeHealth(strings.Join(parts[5:], "\t"))
+		}
+		out[name] = o
 	}
 	return out, nil
 }
 
+// runtimeHealth reads, from a container's state, the runtime's state of its own check: its status and what its last look
+// printed, in a line. Nothing when the container carries no check.
+func runtimeHealth(said string) (string, string) {
+	var state struct {
+		Health *struct {
+			Status string
+			Log    []struct {
+				ExitCode int
+				Output   string
+			}
+		}
+	}
+	if err := json.Unmarshal([]byte(strings.TrimSpace(said)), &state); err != nil || state.Health == nil ||
+		state.Health.Status == "" {
+		return "", ""
+	}
+	h := state.Health
+	last := ""
+	if n := len(h.Log); n > 0 {
+		l := h.Log[n-1]
+		last = firstLine(l.Output)
+		if last == "" && l.ExitCode != 0 {
+			last = fmt.Sprintf("its check exited %d", l.ExitCode)
+		}
+		if len(last) > 200 {
+			last = last[:200] + "…"
+		}
+	}
+	return strings.ToLower(h.Status), last
+}
+
 // missing is an inspect that failed only because a name is not there: what it printed for the others is
 // still the answer.
 func missing(err error) bool {