Liveness alone could not see a web application whose port was open and whose program ran while every request hung for eleven hours (issue 145). A resource now carries the `health` its module declared: the engine makes http and tcp looks itself from the machine to the endpoint's published port, reads a unit's readiness from the show it already makes, hands an exec command or the image's own check to the runtime as the container's check with the declared timing and reads its state from the inspect it already makes, and asks a module's tool on its own node tools. Starting until the check passed, unhealthy once its looks after the grace fail the declared number of times; never more looks than the measured budget; nothing restarted. The statement says contract 2, which tells the controller this engine may be sent the field.
189 lines
6.3 KiB
Go
189 lines
6.3 KiB
Go
package declaration
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Health is how a long-running resource is ready, as the controller composed it from the module's
|
|
// `health` (novox/hq ADR 0240 rule 2, to-be 48 §2–§3, Phase B): one kind and its timing, the endpoint
|
|
// already the port this machine published it on.
|
|
//
|
|
// **The node-engine runs every kind and owns every verdict.** http and tcp it makes itself, from the
|
|
// machine to the port; unit it reads from the service manager it already reads; exec and runtime it hands
|
|
// to the container runtime as the container's own check, with this timing, and reads the state; tool it
|
|
// asks of its own node tools. Nothing else on the machine sets a container's check.
|
|
//
|
|
// Refused here as the controller refuses it near the author, in the same bounds: an engine that took a
|
|
// check it could not judge would say a module ready that nothing looked at.
|
|
type Health struct {
|
|
Kind string `json:"kind"`
|
|
// Endpoint is the module's name for what Port is: for the words a verdict is said in.
|
|
Endpoint string `json:"endpoint,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
Path string `json:"path,omitempty"`
|
|
Status int `json:"status,omitempty"`
|
|
Body string `json:"body,omitempty"`
|
|
Scheme string `json:"scheme,omitempty"`
|
|
Command string `json:"command,omitempty"`
|
|
Tool string `json:"tool,omitempty"`
|
|
Interval string `json:"interval"`
|
|
Timeout string `json:"timeout"`
|
|
Looks int `json:"looks"`
|
|
Grace string `json:"grace"`
|
|
// Needs is the provision the check exercises (to-be 48 §6): said with every verdict, so the
|
|
// controller can hold what it finds under the provider's own condition.
|
|
Needs string `json:"needs,omitempty"`
|
|
}
|
|
|
|
// UnmarshalJSON reads a health strictly, as everything a declaration carries is read: a field this host
|
|
// does not know is a part of the check the controller believes it asked for, and nothing would look at it.
|
|
func (h *Health) UnmarshalJSON(raw []byte) error {
|
|
type plain Health
|
|
var p plain
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(&p); err != nil {
|
|
return fmt.Errorf("health: %w", err)
|
|
}
|
|
*h = Health(p)
|
|
return nil
|
|
}
|
|
|
|
// The kinds.
|
|
const (
|
|
HealthRuntime = "runtime"
|
|
HealthHTTP = "http"
|
|
HealthTCP = "tcp"
|
|
HealthExec = "exec"
|
|
HealthUnit = "unit"
|
|
HealthTool = "tool"
|
|
)
|
|
|
|
// The bounds (ADR 0240 rule 2) — the controller's, held again here.
|
|
const (
|
|
HealthIntervalFloor = 10 * time.Second
|
|
HealthLooksFloor = 2
|
|
HealthWithin = 5 * time.Minute
|
|
)
|
|
|
|
// Every, Within and GraceOf are the timing, read. Validated on arrival, so a parse error here is
|
|
// impossible on a declaration that was accepted; it reads as zero.
|
|
func (h *Health) Every() time.Duration { d, _ := time.ParseDuration(h.Interval); return d }
|
|
func (h *Health) Within() time.Duration { d, _ := time.ParseDuration(h.Timeout); return d }
|
|
func (h *Health) GraceOf() time.Duration { d, _ := time.ParseDuration(h.Grace); return d }
|
|
|
|
// RunByRuntime says the container runtime runs this check as the container's own: exec and runtime.
|
|
func (h *Health) RunByRuntime() bool {
|
|
return h != nil && (h.Kind == HealthExec || h.Kind == HealthRuntime)
|
|
}
|
|
|
|
// Words is the check in a few words, as a verdict is said: "http /healthz on web".
|
|
func (h *Health) Words() string {
|
|
switch h.Kind {
|
|
case HealthHTTP:
|
|
return "http " + h.Path + " on " + orPort(h.Endpoint, h.Port)
|
|
case HealthTCP:
|
|
return "tcp on " + orPort(h.Endpoint, h.Port)
|
|
case HealthTool:
|
|
return "its tool " + h.Tool
|
|
case HealthRuntime:
|
|
return "its image's own check"
|
|
case HealthExec:
|
|
return "its command"
|
|
case HealthUnit:
|
|
return "its unit"
|
|
}
|
|
return h.Kind
|
|
}
|
|
|
|
func orPort(endpoint string, port int) string {
|
|
if endpoint != "" {
|
|
return endpoint
|
|
}
|
|
return fmt.Sprint(port)
|
|
}
|
|
|
|
// problems holds a resource's health to its kind and bounds. container says whether the resource is a
|
|
// container; longRunning whether it stays up.
|
|
func (h *Health) problems(where string, container, longRunning bool) []string {
|
|
if h == nil {
|
|
return nil
|
|
}
|
|
var problems []string
|
|
say := func(format string, args ...any) {
|
|
problems = append(problems, where+": "+fmt.Sprintf(format, args...))
|
|
}
|
|
if !longRunning {
|
|
say("health is judged on what stays up; a step or a scheduled run is judged by its own outcome")
|
|
}
|
|
switch h.Kind {
|
|
case HealthHTTP, HealthTCP:
|
|
if h.Port < 1 || h.Port > 65535 {
|
|
say("a %s check needs the port it looks at", h.Kind)
|
|
}
|
|
case HealthExec:
|
|
if !container {
|
|
say("an exec check runs inside a container")
|
|
}
|
|
if strings.TrimSpace(h.Command) == "" {
|
|
say("an exec check needs a command")
|
|
}
|
|
case HealthRuntime:
|
|
if !container {
|
|
say("a runtime check is a container image's own")
|
|
}
|
|
case HealthUnit:
|
|
if container {
|
|
say("a unit check is a service's or a process's own")
|
|
}
|
|
case HealthTool:
|
|
if strings.TrimSpace(h.Tool) == "" {
|
|
say("a tool check names the tool")
|
|
}
|
|
default:
|
|
say("health of kind %q; it is runtime, http, tcp, exec, unit or tool", h.Kind)
|
|
}
|
|
if h.Kind == HealthHTTP {
|
|
if !strings.HasPrefix(h.Path, "/") {
|
|
say("an http check asks a path starting with /")
|
|
}
|
|
if h.Status != 0 && (h.Status < 100 || h.Status > 599) {
|
|
say("an http check expects status %d, which is not one", h.Status)
|
|
}
|
|
if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" {
|
|
say("an http check is over http or https, not %q", h.Scheme)
|
|
}
|
|
}
|
|
if strings.ContainsAny(h.Command, "\n\r") {
|
|
say("an exec check's command is one line")
|
|
}
|
|
every, everyErr := time.ParseDuration(h.Interval)
|
|
within, withinErr := time.ParseDuration(h.Timeout)
|
|
grace, graceErr := time.ParseDuration(h.Grace)
|
|
switch {
|
|
case everyErr != nil || withinErr != nil || graceErr != nil:
|
|
say("health's interval, timeout and grace are durations")
|
|
default:
|
|
if every < HealthIntervalFloor {
|
|
say("a check looks no more often than every %s, not every %s", HealthIntervalFloor, every)
|
|
}
|
|
if within <= 0 || within >= every {
|
|
say("a look takes more than nothing and less than its interval")
|
|
}
|
|
if grace < 0 {
|
|
say("a grace is not negative")
|
|
}
|
|
if h.Looks >= HealthLooksFloor && grace+time.Duration(h.Looks)*every > HealthWithin {
|
|
say("a grace and the failing looks take at most %s", HealthWithin)
|
|
}
|
|
}
|
|
if h.Looks < HealthLooksFloor {
|
|
say("a check is unhealthy after at least %d failing looks, not %d", HealthLooksFloor, h.Looks)
|
|
}
|
|
return problems
|
|
}
|