mesh-controller/internal/catalogue/health.go

package catalogue

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
)

// A module says how each long-running resource is ready (novox/hq ADR 0240 rule 2, to-be 48 §2 and §8,
// Phase B).
//
// **On the resource, beside its other fields**: one kind and its timing. The node-engine judges every
// long-running resource alive with no declaration (Phase A); this is how a module says what *ready* means
// for one — the image's own check adopted by name, an HTTP request to a declared endpoint, a TCP connect,
// a command in the container, the unit's own readiness, or one of the module's own tools.
//
// **An endpoint is named, never a port or an address**: the check follows the machine's port for that
// endpoint as the endpoint does, so a port this machine gave elsewhere moves the check with it. The
// controller composes the name into the port the machine published it on, and sends the field only to a
// node-engine that reads it (link.ReadinessContract): an older, strict engine refuses a field it does not
// know, whole.
//
// **The bounds are the record's**: an interval not under ten seconds, a timeout under the interval, at
// least two failing looks in a row, and a grace plus the failing looks within five minutes — so a
// resource broken from its start is said within the gate's ten. Refused here, near the author, and again
// by the node-engine, far away, in the same words.

// HealthField is the resource field.
const HealthField = "health"

// The kinds of check (to-be 48 §2).
const (
	HealthRuntime = "runtime"
	HealthHTTP    = "http"
	HealthTCP     = "tcp"
	HealthExec    = "exec"
	HealthUnit    = "unit"
	HealthTool    = "tool"
)

// The bounds and the defaults (ADR 0240 rule 2).
const (
	HealthIntervalDefault = 30 * time.Second
	HealthIntervalFloor   = 10 * time.Second
	HealthTimeoutDefault  = 5 * time.Second
	HealthLooksDefault    = 3
	HealthLooksFloor      = 2
	HealthGraceDefault    = 60 * time.Second
	// HealthWithin is the most a grace and the failing looks may take together: a resource broken
	// from its start is said within the gate's ten minutes with room for its judgings.
	HealthWithin = 5 * time.Minute
)

// HealthRequiredFrom is when `module check` refuses a long-running resource without `health` (ADR 0240
// rule 8): six weeks after liveness was first judged live (2026-10-07), unless the catalogue's count of
// undeclared resources reached zero first — which its own counter enforces by never letting it rise.
var HealthRequiredFrom = time.Date(2026, 11, 18, 0, 0, 0, 0, time.UTC)

// Health is one resource's declaration, read.
type Health struct {
	Kind string
	// Endpoint is the `listens` name an http or tcp check looks at.
	Endpoint string
	// Path, Status, Body and Scheme are an http check's: the path asked, the status expected (zero: any
	// status under 400), a text the answer must hold, and http or https.
	Path   string
	Status int
	Body   string
	Scheme string
	// Command is an exec check's command, run by the container's shell.
	Command string
	// Tool is a tool check's tool, one of the module's own.
	Tool string
	// The timing, with the defaults applied.
	Interval, Timeout, Grace time.Duration
	Looks                    int
	// Needs is the provision the check exercises (to-be 48 §6): while its provider for this consumer is
	// unhealthy, what this check finds is held under the provider's condition.
	Needs string
}

// healthKeys are the keys a `health` field may carry; anything else is refused by name.
var healthKeys = map[string]bool{"kind": true, "endpoint": true, "path": true, "status": true, "body": true,
	"scheme": true, "command": true, "tool": true, "interval": true, "timeout": true, "looks": true,
	"grace": true, "needs": true}

// healthAddressKeys are what a check may not be aimed by: a port or an address does not follow the
// machine's port for the endpoint, and a manifest is the same on every machine.
var healthAddressKeys = map[string]bool{"port": true, "address": true, "host": true, "url": true, "ip": true}

// LongRunning says whether a manifest resource stays up: a container that is no step and on no schedule,
// a service stated running, a process that is no step and on no schedule (ADR 0240 rule 1).
func LongRunning(r map[string]any) bool {
	switch fmt.Sprint(r["type"]) {
	case "container", "process":
		if once, _ := r["run-once"].(bool); once {
			return false
		}
		return r["schedule"] == nil
	case "service":
		return fmt.Sprint(r["state"]) == "running"
	}
	return false
}

// ReadHealth reads a resource's `health` field, defaults applied. False when it carries none.
func ReadHealth(r map[string]any) (Health, bool, []string) {
	raw, present := r[HealthField]
	if !present {
		return Health{}, false, nil
	}
	id := fmt.Sprint(r["id"])
	fields, ok := raw.(map[string]any)
	if !ok {
		return Health{}, true, []string{fmt.Sprintf("%s: health is %T; it is an object with a kind and its timing", id, raw)}
	}
	var problems []string
	keys := make([]string, 0, len(fields))
	for k := range fields {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		switch {
		case healthAddressKeys[k]:
			problems = append(problems, fmt.Sprintf("%s: its health names a %s; a check names an endpoint the module "+
				"declares under listens, by its name, so it follows the port this machine gives it (ADR 0240 rule 2)", id, k))
		case !healthKeys[k]:
			problems = append(problems, fmt.Sprintf("%s: its health says %q, which a health check does not have", id, k))
		}
	}
	text := func(key string) string {
		v, present := fields[key]
		if !present {
			return ""
		}
		s, ok := v.(string)
		if !ok {
			problems = append(problems, fmt.Sprintf("%s: its health's %s is %T; it is text", id, key, v))
		}
		return strings.TrimSpace(s)
	}
	duration := func(key string, fallback time.Duration) time.Duration {
		s := text(key)
		if s == "" {
			return fallback
		}
		d, err := time.ParseDuration(s)
		if err != nil || d < 0 {
			problems = append(problems, fmt.Sprintf("%s: its health's %s is %q; it is a duration such as \"30s\"", id, key, s))
			return fallback
		}
		return d
	}
	number := func(key string, fallback int) int {
		v, present := fields[key]
		if !present {
			return fallback
		}
		f, ok := v.(float64)
		if !ok || f != float64(int(f)) {
			problems = append(problems, fmt.Sprintf("%s: its health's %s is %v; it is a whole number", id, key, v))
			return fallback
		}
		return int(f)
	}
	h := Health{Kind: text("kind"), Endpoint: text("endpoint"), Path: text("path"), Body: text("body"),
		Scheme: text("scheme"), Command: text("command"), Tool: text("tool"), Needs: text("needs"),
		Interval: duration("interval", HealthIntervalDefault), Timeout: duration("timeout", HealthTimeoutDefault),
		Grace: duration("grace", HealthGraceDefault), Looks: number("looks", HealthLooksDefault),
		Status: number("status", 0)}
	return h, true, problems
}

// healthProblems is everything wrong with a manifest's `health` fields, in the manifest's words (to-be
// 48 §8): a field on something that does not stay up, a kind its resource cannot have, an endpoint the
// module does not declare, a port or an address, a timing outside its bounds, a tool the module does not
// serve, and a tool check with no check of another kind beside it on the module.
func healthProblems(m Manifest) []string {
	var problems []string
	endpoints := map[string]Listening{}
	for _, l := range m.Listens {
		if n := strings.TrimSpace(l.Name); n != "" {
			endpoints[n] = l
		}
	}
	tools := map[string]bool{}
	for _, t := range m.Tools {
		tools[t] = true
	}
	wants := map[string]bool{}
	for _, w := range m.Wants() {
		wants[w] = true
	}
	var toolChecks []string
	otherKinds := 0
	resources := m.Resources
	// A module's own runtime is judged as the process the controller writes for it (novox/hq ADR 0366 §1),
	// and only a module that runs as its own account has one.
	own, hasOwn := OwnRuntimeHealth(m)
	if hasOwn {
		if m.RunsAs == "" {
			problems = append(problems, fmt.Sprintf("%s declares runtime-health and runs as no account of its own "+
				"(runs-as): only a module served by a runtime of its own has that runtime's health to declare "+
				"(novox/hq ADR 0366 §1)", m.Module))
			hasOwn = false
		} else {
			resources = append(append([]map[string]any(nil), m.Resources...), own)
		}
	}
	for i, r := range resources {
		h, has, read := ReadHealth(r)
		if !has {
			continue
		}
		ownRuntime := hasOwn && i == len(resources)-1
		id, typ := fmt.Sprint(r["id"]), fmt.Sprint(r["type"])
		where := m.Module + ": " + id
		if ownRuntime {
			where = m.Module + ": its runtime-health"
		}
		for _, p := range read {
			problems = append(problems, m.Module+": "+p)
		}
		if !LongRunning(r) {
			problems = append(problems, fmt.Sprintf("%s declares health and does not stay up: a step, anything on a "+
				"schedule and a service not stated running are judged by their step and their schedule (ADR 0240 rule 1)", where))
			continue
		}
		switch h.Kind {
		case HealthRuntime, HealthExec:
			if typ != "container" {
				problems = append(problems, fmt.Sprintf("%s is a %s and its health is %q, which only a container has: "+
					"the runtime runs it inside the container", where, typ, h.Kind))
			}
		case HealthUnit:
			if typ == "container" {
				problems = append(problems, fmt.Sprintf("%s is a container and its health is %q, which is a service's "+
					"or a process's own readiness", where, h.Kind))
			}
		case HealthHTTP, HealthTCP, HealthTool:
		case "":
			problems = append(problems, fmt.Sprintf("%s declares health with no kind: %s", where, healthKindsWords()))
		default:
			problems = append(problems, fmt.Sprintf("%s declares health of kind %q: %s", where, h.Kind, healthKindsWords()))
		}
		switch h.Kind {
		case HealthHTTP, HealthTCP:
			l, declared := endpoints[h.Endpoint]
			switch {
			case h.Endpoint == "":
				problems = append(problems, fmt.Sprintf("%s's %s check names no endpoint: it names one the module "+
					"declares under listens, by its name", where, h.Kind))
			case !declared:
				problems = append(problems, fmt.Sprintf("%s's %s check names the endpoint %q, which %s does not declare "+
					"under listens (%s)", where, h.Kind, h.Endpoint, m.Module, namedEndpointsWords(endpoints)))
			case l.At() != "tcp":
				problems = append(problems, fmt.Sprintf("%s's %s check names %q, which is %s: a check connects over tcp",
					where, h.Kind, h.Endpoint, l.At()))
			}
		default:
			if h.Endpoint != "" {
				problems = append(problems, fmt.Sprintf("%s's %s check names an endpoint, which only an http or tcp check "+
					"looks at", where, h.Kind))
			}
		}
		if h.Kind != HealthHTTP && (h.Path != "" || h.Status != 0 || h.Body != "" || h.Scheme != "") {
			problems = append(problems, fmt.Sprintf("%s's %s check says a path, a status, a body or a scheme, which "+
				"only an http check has", where, h.Kind))
		}
		if h.Kind == HealthHTTP {
			if h.Path != "" && !strings.HasPrefix(h.Path, "/") {
				problems = append(problems, fmt.Sprintf("%s's http check asks %q; a path starts with /", where, h.Path))
			}
			if h.Status != 0 && (h.Status < 100 || h.Status > 599) {
				problems = append(problems, fmt.Sprintf("%s's http check expects status %d, which is not one", where, h.Status))
			}
			if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" {
				problems = append(problems, fmt.Sprintf("%s's http check is over %q; it is http or https", where, h.Scheme))
			}
		}
		if (h.Command != "") != (h.Kind == HealthExec) {
			if h.Kind == HealthExec {
				problems = append(problems, fmt.Sprintf("%s's exec check says no command", where))
			} else {
				problems = append(problems, fmt.Sprintf("%s's %s check says a command, which only an exec check runs",
					where, h.Kind))
			}
		}
		if h.Kind == HealthTool {
			switch {
			case h.Tool == "":
				problems = append(problems, fmt.Sprintf("%s's tool check names no tool", where))
			case !tools[h.Tool]:
				problems = append(problems, fmt.Sprintf("%s's tool check asks %q, which %s does not serve (its tools: %s)",
					where, h.Tool, m.Module, orNoneWords(m.Tools)))
			}
			// Its own runtime being alive, judged by the node-engine, is the check the module does not run
			// itself that this tool check stands beside (novox/hq ADR 0366 §2).
			if !ownRuntime {
				toolChecks = append(toolChecks, id)
			}
		} else {
			if h.Tool != "" {
				problems = append(problems, fmt.Sprintf("%s's %s check names a tool, which only a tool check asks", where, h.Kind))
			}
			if h.Kind != "" {
				otherKinds++
			}
		}
		if h.Needs != "" && !wants[h.Needs] {
			problems = append(problems, fmt.Sprintf("%s's check needs %q, which %s does not require: a check names "+
				"the provision it exercises, among those the module requires", where, h.Needs, m.Module))
		}
		problems = append(problems, healthTimingProblems(where, h)...)
	}
	if len(toolChecks) > 0 && otherKinds == 0 {
		problems = append(problems, fmt.Sprintf("%s judges itself only by its own tool (%s): a tool check is for "+
			"function no endpoint shows, and only beside a check of another kind the module does not run itself "+
			"(ADR 0227 rule 8, ADR 0240 rule 2)", m.Module, strings.Join(toolChecks, ", ")))
	}
	return problems
}

// healthTimingProblems holds a check's timing to its bounds.
func healthTimingProblems(where string, h Health) []string {
	var problems []string
	if h.Interval < HealthIntervalFloor {
		problems = append(problems, fmt.Sprintf("%s looks every %s; a check looks no more often than every %s — "+
			"the mesh is a guest on the machine (ADR 0240 rule 2)", where, h.Interval, HealthIntervalFloor))
	}
	if h.Timeout <= 0 || h.Timeout >= h.Interval {
		problems = append(problems, fmt.Sprintf("%s gives a look %s, which must be more than nothing and under its "+
			"interval of %s", where, h.Timeout, h.Interval))
	}
	if h.Looks < HealthLooksFloor {
		problems = append(problems, fmt.Sprintf("%s is unhealthy after %d failing look(s); it is at least %d — one "+
			"look can be wrong (issue 277)", where, h.Looks, HealthLooksFloor))
	}
	if h.Grace < 0 {
		problems = append(problems, fmt.Sprintf("%s has a grace of %s", where, h.Grace))
	}
	if h.Looks >= HealthLooksFloor && h.Interval >= HealthIntervalFloor {
		if took := h.Grace + time.Duration(h.Looks)*h.Interval; took > HealthWithin {
			problems = append(problems, fmt.Sprintf("%s is said unhealthy at the earliest %s after it starts (a grace "+
				"of %s and %d looks every %s); it is at most %s, so a resource broken from its start is said within "+
				"the gate's bound", where, took, h.Grace, h.Looks, h.Interval, HealthWithin))
		}
	}
	return problems
}

func healthKindsWords() string {
	return "a check is runtime (the image's own, adopted by name), http, tcp, exec, unit or tool"
}

func namedEndpointsWords(endpoints map[string]Listening) string {
	if len(endpoints) == 0 {
		return "it names none"
	}
	names := make([]string, 0, len(endpoints))
	for n := range endpoints {
		names = append(names, n)
	}
	sort.Strings(names)
	return "it names " + strings.Join(names, ", ")
}

func orNoneWords(names []string) string {
	if len(names) == 0 {
		return "none"
	}
	return strings.Join(names, ", ")
}

// Undeclared is every long-running resource of the manifest without `health`, by id (ADR 0240 rule 8):
// what the catalogue's count counts.
func Undeclared(m Manifest) []string {
	var out []string
	for _, r := range m.Resources {
		if !LongRunning(r) {
			continue
		}
		if _, has := r[HealthField]; !has {
			out = append(out, fmt.Sprint(r["id"]))
		}
	}
	return out
}

// HealthChecks is every tool a module's health asks, as `<module>.<tool>`: what a machine's node-engine
// is granted to ask of its own node tools (to-be 48 §3).
func HealthChecks(m Manifest) []string {
	var out []string
	resources := m.Resources
	if own, has := OwnRuntimeHealth(m); has && m.RunsAs != "" {
		resources = append(append([]map[string]any(nil), resources...), own)
	}
	for _, r := range resources {
		h, has, _ := ReadHealth(r)
		if has && h.Kind == HealthTool && h.Tool != "" && LongRunning(r) {
			out = append(out, m.Module+"."+h.Tool)
		}
	}
	sort.Strings(out)
	return out
}

// OwnRuntimeHealth is a module's `runtime-health` as the resource it is put on (novox/hq ADR 0366 §1): the
// process `<module>.own-runtime`, carrying it as its `health`. False when the module declares none.
func OwnRuntimeHealth(m Manifest) (map[string]any, bool) {
	if m.RuntimeHealth == nil {
		return nil, false
	}
	return map[string]any{"id": m.Module + "." + OwnRuntimeID(), "type": "process", HealthField: m.RuntimeHealth}, true
}

// healthInto composes a resource's `health` into the node-engine's words, or takes it away (to-be 48 §2,
// §3): for an engine that reads it, the endpoint becomes the port this machine published it on and the
// defaults are written out; for one that does not — older and strict — the field is not sent, and the
// resource is judged by liveness alone, as before.
func healthInto(resource map[string]any, m Manifest, with Rendering) {
	if _, has := resource[HealthField]; !has {
		return
	}
	if !with.ReadsHealth {
		delete(resource, HealthField)
		return
	}
	h, _, _ := ReadHealth(resource)
	out := map[string]any{"kind": h.Kind, "interval": h.Interval.String(), "timeout": h.Timeout.String(),
		"looks": h.Looks, "grace": h.Grace.String()}
	if h.Endpoint != "" {
		out["endpoint"] = h.Endpoint
		if port, ok := EndpointPort(m, h.Endpoint); ok {
			out["port"] = with.machinePort(m.Module, port)
		}
	}
	if h.Kind == HealthHTTP {
		path := h.Path
		if path == "" {
			path = "/"
		}
		out["path"] = path
		if h.Status != 0 {
			out["status"] = h.Status
		}
		if h.Body != "" {
			out["body"] = h.Body
		}
		if h.Scheme != "" {
			out["scheme"] = h.Scheme
		}
	}
	if h.Command != "" {
		out["command"] = h.Command
	}
	if h.Tool != "" {
		out["tool"] = h.Tool
	}
	if h.Needs != "" {
		out["needs"] = h.Needs
	}
	resource[HealthField] = out
}

// HealthWords is a declared check in a line, for `module check` and `node show`.
func HealthWords(h Health) string {
	var what string
	switch h.Kind {
	case HealthHTTP:
		what = "http " + orSlash(h.Path) + " on " + h.Endpoint
		if h.Status != 0 {
			what += " expecting " + strconv.Itoa(h.Status)
		}
	case HealthTCP:
		what = "tcp on " + h.Endpoint
	case HealthTool:
		what = "its tool " + h.Tool
	case HealthRuntime:
		what = "its image's own check"
	default:
		what = h.Kind
	}
	return fmt.Sprintf("%s every %s", what, h.Interval)
}

func orSlash(p string) string {
	if p == "" {
		return "/"
	}
	return p
}