From 468d509462fe3022f7c93bf7874295eeeb9db385 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 7 Oct 2026 14:39:13 +0200 Subject: [PATCH] Read how a module says each resource is ready, and send it to engines that read it (hq ADR 0240, to-be 48 Phase B) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A module could say nothing about what ready means for what it runs, so a web application with its port open and its requests hanging passed everything for eleven hours (issue 145). A long-running resource now carries `health` — the image's own check adopted by name, http, tcp, exec, unit or a module's own tool, with its timing — refused near its author when it names a port or an address, an endpoint the module does not declare, a tool it does not serve, a tool check alone, or a timing outside the record's bounds. It is composed with the endpoint as the port this machine published it on, and sent only to a node-engine whose statement says it reads it: an older one would refuse the whole declaration. The engine is granted its own machine's instance of each health tool. `module check` warns of every long-running resource without `health`, counts them for the catalogue, and refuses them from 2026-11-18. A check's findings stay out of a condition's summary. The node-engine's validator is vendored at its Phase B commit, so what is composed is judged by the words the engine takes. --- cmd/mesh-controller/check.go | 46 ++ cmd/mesh-controller/health_check_test.go | 47 ++ cmd/mesh-controller/module_health.go | 8 +- cmd/mesh-controller/module_health_test.go | 33 ++ cmd/mesh-controller/plan.go | 17 + cmd/mesh-controller/replays_test.go | 55 +++ go.mod | 2 +- go.sum | 2 + internal/broker/nats.go | 25 + internal/broker/users.go | 7 +- internal/broker/witness_grants_test.go | 28 ++ internal/catalogue/declaration.go | 9 + internal/catalogue/health.go | 460 ++++++++++++++++++ internal/catalogue/health_test.go | 217 +++++++++ internal/catalogue/manifest.go | 3 + internal/inventory/busrecords.go | 2 + internal/inventory/health.go | 4 + internal/link/protocol.go | 9 + internal/link/protocol_test.go | 5 +- .../internal/declaration/declaration.go | 44 +- .../mesh-host/internal/declaration/health.go | 188 +++++++ vendor/modules.txt | 4 +- 22 files changed, 1205 insertions(+), 10 deletions(-) create mode 100644 cmd/mesh-controller/health_check_test.go create mode 100644 internal/catalogue/health.go create mode 100644 internal/catalogue/health_test.go create mode 100644 vendor/github.com/novox/mesh-host/internal/declaration/health.go diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 463af4d..31c0eec 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/novox/mesh-controller/internal/catalogue" ) @@ -107,6 +108,28 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { names = append(names, name) } sort.Strings(names) + + // **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the + // date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a + // change lower it, never raise it. + undeclared := 0 + required := !checkNow().Before(catalogue.HealthRequiredFrom) + for _, name := range names { + missing := catalogue.Undeclared(shelf[name]) + undeclared += len(missing) + if len(missing) == 0 { + continue + } + if required { + for _, id := range missing { + fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+ + "health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02")) + } + failed += len(missing) + faulted[name] = true + } + } + for _, name := range names { m := shelf[name] if faulted[name] { @@ -138,8 +161,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { } fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", ")) } + // How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone. + var checks []string + for _, r := range m.Resources { + if h, has, _ := catalogue.ReadHealth(r); has { + checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h))) + } + } + if len(checks) > 0 { + fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; ")) + } + if missing := catalogue.Undeclared(m); len(missing) > 0 { + fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+ + "refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02")) + } fmt.Fprintln(out) } + // The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads. + fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared) if failed > 0 { return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths)) } @@ -150,6 +189,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { return nil } +// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without +// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps. +const UndeclaredHealthLine = "long-running resources without health:" + +// checkNow is the clock `module check` judges the date by; a test sets it. +var checkNow = time.Now + func joinInvokes(invokes []string) string { if len(invokes) == 1 && invokes[0] == "*" { return "every tool" diff --git a/cmd/mesh-controller/health_check_test.go b/cmd/mesh-controller/health_check_test.go new file mode 100644 index 0000000..6f3f235 --- /dev/null +++ b/cmd/mesh-controller/health_check_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule +// 8): `module check` warns and counts the undeclared before the date, and refuses them from it. +func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) { + dir := t.TempDir() + digest := "@sha256:" + strings.Repeat("a", 64) + path := filepath.Join(dir, "module.json") + os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[ + {"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"], + "health":{"kind":"http","endpoint":"web"}}, + {"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"}, + {"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600) + defer func() { checkNow = time.Now }() + + checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) } + var out bytes.Buffer + if err := moduleCheck([]string{path}, &out); err != nil { + t.Fatalf("refused before the date: %v\n%s", err, out.String()) + } + for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up", + catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} { + if !strings.Contains(out.String(), want) { + t.Errorf("the check does not say %q:\n%s", want, out.String()) + } + } + + checkNow = func() time.Time { return catalogue.HealthRequiredFrom } + out.Reset() + if err := moduleCheck([]string{path}, &out); err == nil { + t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String()) + } + if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") { + t.Errorf("the refusal does not name the resource:\n%s", out.String()) + } +} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 5529be9..94ea528 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -71,7 +71,8 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke resources := make([]inventory.ResourceHealth, 0, len(h.Resources)) for _, r := range h.Resources { kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, - State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts} + State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts, + Check: r.Check, Needs: r.Needs} resources = append(resources, kept) if r.State == link.StateUnhealthy && r.Module != "" { unhealthy[r.Module] = append(unhealthy[r.Module], kept) @@ -172,6 +173,11 @@ func reasonWords(r inventory.ResourceHealth) string { case "": return "is unhealthy" } + // What a declared check found says an endpoint, a path or an address: evidence, never the summary the + // operator's channel carries (ADR 0234 §6). The summary names the check. + if r.Check != "" { + return "fails its " + r.Check + " check" + } return "is unhealthy: " + r.Reason } diff --git a/cmd/mesh-controller/module_health_test.go b/cmd/mesh-controller/module_health_test.go index 7eb4c03..dba6538 100644 --- a/cmd/mesh-controller/module_health_test.go +++ b/cmd/mesh-controller/module_health_test.go @@ -161,3 +161,36 @@ func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) { t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err) } } + +// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240 +// Phase B): an older engine is strict and would refuse the whole declaration for the field. +func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + reads := func() bool { + t.Helper() + got, err := engineReadsHealth(ctx, inv, "anchor") + if err != nil { + t.Fatal(err) + } + return got + } + if reads() { + t.Fatal("an engine that never stated anything is sent health") + } + if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil { + t.Fatal(err) + } + if reads() { + t.Fatal("an engine judging liveness alone is sent health") + } + later := aStatement(h0.Add(time.Minute), link.StateHealthy) + later.Contract = link.ReadinessContract + if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if !reads() { + t.Fatal("an engine that reads health is not sent it") + } +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index de31ed0..e531afd 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -17,6 +17,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/overlay" ) @@ -860,7 +861,12 @@ func renderingFor(ctx context.Context, open *stores, node string, if artifactStore != "" { reach = map[string]string{"mesh-artifact-store": artifactStore} } + readsHealth, err := engineReadsHealth(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } return catalogue.Rendering{ + ReadsHealth: readsHealth, BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, @@ -872,6 +878,17 @@ func renderingFor(ctx context.Context, open *stores, node string, }, record, nil } +// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240 +// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the +// field, because it parses strictly and would refuse the whole declaration for it. +func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.ReadinessContract, nil +} + // zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR // 0199): the zone settled from that node's settings, the node's private address, the port the // answering listen is published on there. diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index dc0d0d1..aeaabe1 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -286,3 +287,57 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) } } + +// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq +// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its +// program ran while every request hung, and the mesh said its machine was healthy; a person found it. +// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check +// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it +// found looking at such a program; here the controller hears that statement on two looks in a row and +// raises the module's condition — the second, never the first. `null` is an engine older than the +// readiness check: it states the program alive, and nothing is raised. +func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + + `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + + `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) + if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the engine's statement of the silent web application: %v", err) + } + silent = raw + } + var h link.Health + if err := json.Unmarshal(silent, &h); err != nil { + t.Fatal(err) + } + if h.Contract == 0 { + t.Fatal("the engine states nothing of the silent web application: it is older than the judging") + } + open1 := func() []conditions.Condition { + t.Helper() + list, err := conditionsFrom.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + for look := 1; look <= 2; look++ { + at := time.Now().Add(time.Duration(look) * time.Second) + h.At = at + if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { + t.Fatal(err) + } + raised := open1() + switch { + case look == 1 && len(raised) != 0: + t.Fatalf("one look raised %v", raised[0].Key) + case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): + t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) + case look == 2: + t.Logf("raised on the second look: %s", raised[0].Summary) + } + } +} diff --git a/go.mod b/go.mod index 81aec91..79ec0e3 100644 --- a/go.mod +++ b/go.mod @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac diff --git a/go.sum b/go.sum index 0b80baf..87fafd1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e h1:g9h4QRaAMg5yaJLwqtb0FoOs23DVGUYpW6qvnQ3oY5A= git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs= +git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac h1:KvnKtJ2rWeIE/t4GweK+JL0OjKSNxsrVP3/nMdpii8o= +git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac/go.mod h1:VlilMCRZ5yyNXg7SNigNBLr0Gt32jrGw5KSNq5JAVYs= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 44f7343..cdca1d3 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -122,6 +122,10 @@ type Principal struct { // key, and nothing else of the bucket, so it can judge a new controller build and put the previous // one back. WitnessesController bool + // Checks are the tools a machine principal's node-engine asks as a declared health check, each + // `.` (novox/hq ADR 0240, to-be 48 §3): asked of the instance on its own machine and + // nowhere else. + Checks []string // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal // and never appears here: this file is written to a node's disk and read by a server, and a @@ -1021,5 +1025,26 @@ func WitnessSubjects(p Principal) []string { if p.WitnessesController { out = append(out, lease.LeaseReadSubject(LeaseBucket)) } + return append(out, CheckSubjects(p)...) +} + +// CheckSubjects are the tools a machine's node-engine asks as declared health checks (novox/hq ADR 0240, +// to-be 48 §3): each `.` on this machine's instance — `mesh.mod..tool..` +// — and never the plain subject, which any machine's instance may answer. Sorted and once each. +func CheckSubjects(p Principal) []string { + seen := map[string]bool{} + var out []string + for _, c := range p.Checks { + module, tool, ok := strings.Cut(c, ".") + if !ok || !safeSubject.MatchString(module) || !safeSubject.MatchString(tool) { + continue + } + subject := "mesh.mod." + module + ".tool." + tool + "." + p.Node + if !seen[subject] { + seen[subject] = true + out = append(out, subject) + } + } + sort.Strings(out) return out } diff --git a/internal/broker/users.go b/internal/broker/users.go index d4534b0..a69474f 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -44,6 +44,9 @@ type Declared struct { // SnapshotsTheBus says the module holds mesh-broker — it is the bus — and so is the one module // granted the snapshot API, to copy the bus's streams for the night's backup (novox/hq ADR 0235). SnapshotsTheBus bool + // Checks are the module's own tools its health asks, each `.` (novox/hq ADR 0240, to-be + // 48 §3): the machine's node-engine asks them of its own node tools, and is granted that and no more. + Checks []string } // Records is what composing a user list needs to know about the mesh, and nothing more. @@ -73,12 +76,14 @@ func Users(r Records) ([]Principal, error) { for _, node := range sortedCopy(r.Nodes) { witness := false + var checks []string for _, d := range r.Assigned[node] { if d.Module == controllerModule { witness = true } + checks = append(checks, d.Checks...) } - out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness}) + out = append(out, Principal{Kind: KindNode, Node: node, WitnessesController: witness, Checks: checks}) // **Where the runtime is assigned, the machine gets one runtime principal in place of the // runtime module's own** (novox/hq ADR 0175, to-be 38). It carries every module on the // node: its serving grants are the union of theirs. Every other module keeps its own diff --git a/internal/broker/witness_grants_test.go b/internal/broker/witness_grants_test.go index e2b1538..91932a7 100644 --- a/internal/broker/witness_grants_test.go +++ b/internal/broker/witness_grants_test.go @@ -35,3 +35,31 @@ func TestTheWitnessIsGrantedWhatItReadsAndNoMore(t *testing.T) { } } } + +// A module's health that asks one of its own tools is asked by the machine's node-engine, of the instance +// on its own machine and nowhere else (novox/hq ADR 0240, to-be 48 §3). +func TestTheEngineIsGrantedTheToolsItsModulesHealthAsks(t *testing.T) { + users, err := Users(Records{Nodes: []string{"control", "edge"}, + Assigned: map[string][]Declared{"edge": {{Module: "keycloak", Checks: []string{"keycloak.keycloak_admin_health"}}}}}) + if err != nil { + t.Fatal(err) + } + for _, u := range users { + if u.Kind != KindNode { + continue + } + perms, err := PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + mine := "mesh.mod.keycloak.tool.keycloak_admin_health.edge" + if got := slices.Contains(perms.Publish, mine); got != (u.Node == "edge") { + t.Errorf("%s may ask keycloak's health tool on edge: %v", u.Node, got) + } + for _, p := range perms.Publish { + if p == "mesh.mod.keycloak.tool.keycloak_admin_health" || p == "mesh.mod.keycloak.tool.>" { + t.Errorf("%s may ask the tool of any machine: %s", u.Node, p) + } + } + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 7a2774c..0024654 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -235,6 +235,12 @@ type Rendering struct { // mounts and environment. A node setting fixed at installation; empty means the default, // /var/lib — see dir_into.go. DataRoot string + + // ReadsHealth says this machine's node-engine reads a resource's `health` (novox/hq ADR 0240 Phase B: + // its statement's contract is link.ReadinessContract or later). An older engine parses strictly and + // refuses a field it does not know, whole — so to it the field is not sent, and what it runs is + // judged by liveness alone. + ReadsHealth bool } // machinePort is where a module's port lives on this machine, or the port itself when the mesh has @@ -957,6 +963,9 @@ func (r Resolution) compose(with Rendering, owner map[string]string, return nil, err } publishedOn(copied, m.Module, with) + // How it is ready, in the node-engine's words: its endpoint as the port this machine + // published it on — or not sent at all to an engine older than the field (ADR 0240). + healthInto(copied, m, with) // The account's environment and every module's shell code, where this module holds the // seat that places them (novox/hq ADR 0203, ADR 0204). Gathered from every module on // the node, as the jails are, and **last of every placeholder pass**: shell code is a diff --git a/internal/catalogue/health.go b/internal/catalogue/health.go new file mode 100644 index 0000000..cbfc298 --- /dev/null +++ b/internal/catalogue/health.go @@ -0,0 +1,460 @@ +package catalogue + +import ( + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// A module says how each long-running resource is ready (novox/hq ADR 0240 rule 2, to-be 48 §2 and §8, +// Phase B). +// +// **On the resource, beside its other fields**: one kind and its timing. The node-engine judges every +// long-running resource alive with no declaration (Phase A); this is how a module says what *ready* means +// for one — the image's own check adopted by name, an HTTP request to a declared endpoint, a TCP connect, +// a command in the container, the unit's own readiness, or one of the module's own tools. +// +// **An endpoint is named, never a port or an address**: the check follows the machine's port for that +// endpoint as the endpoint does, so a port this machine gave elsewhere moves the check with it. The +// controller composes the name into the port the machine published it on, and sends the field only to a +// node-engine that reads it (link.ReadinessContract): an older, strict engine refuses a field it does not +// know, whole. +// +// **The bounds are the record's**: an interval not under ten seconds, a timeout under the interval, at +// least two failing looks in a row, and a grace plus the failing looks within five minutes — so a +// resource broken from its start is said within the gate's ten. Refused here, near the author, and again +// by the node-engine, far away, in the same words. + +// HealthField is the resource field. +const HealthField = "health" + +// The kinds of check (to-be 48 §2). +const ( + HealthRuntime = "runtime" + HealthHTTP = "http" + HealthTCP = "tcp" + HealthExec = "exec" + HealthUnit = "unit" + HealthTool = "tool" +) + +// The bounds and the defaults (ADR 0240 rule 2). +const ( + HealthIntervalDefault = 30 * time.Second + HealthIntervalFloor = 10 * time.Second + HealthTimeoutDefault = 5 * time.Second + HealthLooksDefault = 3 + HealthLooksFloor = 2 + HealthGraceDefault = 60 * time.Second + // HealthWithin is the most a grace and the failing looks may take together: a resource broken + // from its start is said within the gate's ten minutes with room for its judgings. + HealthWithin = 5 * time.Minute +) + +// HealthRequiredFrom is when `module check` refuses a long-running resource without `health` (ADR 0240 +// rule 8): six weeks after liveness was first judged live (2026-10-07), unless the catalogue's count of +// undeclared resources reached zero first — which its own counter enforces by never letting it rise. +var HealthRequiredFrom = time.Date(2026, 11, 18, 0, 0, 0, 0, time.UTC) + +// Health is one resource's declaration, read. +type Health struct { + Kind string + // Endpoint is the `listens` name an http or tcp check looks at. + Endpoint string + // Path, Status, Body and Scheme are an http check's: the path asked, the status expected (zero: any + // status under 400), a text the answer must hold, and http or https. + Path string + Status int + Body string + Scheme string + // Command is an exec check's command, run by the container's shell. + Command string + // Tool is a tool check's tool, one of the module's own. + Tool string + // The timing, with the defaults applied. + Interval, Timeout, Grace time.Duration + Looks int + // Needs is the provision the check exercises (to-be 48 §6): while its provider for this consumer is + // unhealthy, what this check finds is held under the provider's condition. + Needs string +} + +// healthKeys are the keys a `health` field may carry; anything else is refused by name. +var healthKeys = map[string]bool{"kind": true, "endpoint": true, "path": true, "status": true, "body": true, + "scheme": true, "command": true, "tool": true, "interval": true, "timeout": true, "looks": true, + "grace": true, "needs": true} + +// healthAddressKeys are what a check may not be aimed by: a port or an address does not follow the +// machine's port for the endpoint, and a manifest is the same on every machine. +var healthAddressKeys = map[string]bool{"port": true, "address": true, "host": true, "url": true, "ip": true} + +// LongRunning says whether a manifest resource stays up: a container that is no step and on no schedule, +// a service stated running, a process that is no step and on no schedule (ADR 0240 rule 1). +func LongRunning(r map[string]any) bool { + switch fmt.Sprint(r["type"]) { + case "container", "process": + if once, _ := r["run-once"].(bool); once { + return false + } + return r["schedule"] == nil + case "service": + return fmt.Sprint(r["state"]) == "running" + } + return false +} + +// ReadHealth reads a resource's `health` field, defaults applied. False when it carries none. +func ReadHealth(r map[string]any) (Health, bool, []string) { + raw, present := r[HealthField] + if !present { + return Health{}, false, nil + } + id := fmt.Sprint(r["id"]) + fields, ok := raw.(map[string]any) + if !ok { + return Health{}, true, []string{fmt.Sprintf("%s: health is %T; it is an object with a kind and its timing", id, raw)} + } + var problems []string + keys := make([]string, 0, len(fields)) + for k := range fields { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + switch { + case healthAddressKeys[k]: + problems = append(problems, fmt.Sprintf("%s: its health names a %s; a check names an endpoint the module "+ + "declares under listens, by its name, so it follows the port this machine gives it (ADR 0240 rule 2)", id, k)) + case !healthKeys[k]: + problems = append(problems, fmt.Sprintf("%s: its health says %q, which a health check does not have", id, k)) + } + } + text := func(key string) string { + v, present := fields[key] + if !present { + return "" + } + s, ok := v.(string) + if !ok { + problems = append(problems, fmt.Sprintf("%s: its health's %s is %T; it is text", id, key, v)) + } + return strings.TrimSpace(s) + } + duration := func(key string, fallback time.Duration) time.Duration { + s := text(key) + if s == "" { + return fallback + } + d, err := time.ParseDuration(s) + if err != nil || d < 0 { + problems = append(problems, fmt.Sprintf("%s: its health's %s is %q; it is a duration such as \"30s\"", id, key, s)) + return fallback + } + return d + } + number := func(key string, fallback int) int { + v, present := fields[key] + if !present { + return fallback + } + f, ok := v.(float64) + if !ok || f != float64(int(f)) { + problems = append(problems, fmt.Sprintf("%s: its health's %s is %v; it is a whole number", id, key, v)) + return fallback + } + return int(f) + } + h := Health{Kind: text("kind"), Endpoint: text("endpoint"), Path: text("path"), Body: text("body"), + Scheme: text("scheme"), Command: text("command"), Tool: text("tool"), Needs: text("needs"), + Interval: duration("interval", HealthIntervalDefault), Timeout: duration("timeout", HealthTimeoutDefault), + Grace: duration("grace", HealthGraceDefault), Looks: number("looks", HealthLooksDefault), + Status: number("status", 0)} + return h, true, problems +} + +// healthProblems is everything wrong with a manifest's `health` fields, in the manifest's words (to-be +// 48 §8): a field on something that does not stay up, a kind its resource cannot have, an endpoint the +// module does not declare, a port or an address, a timing outside its bounds, a tool the module does not +// serve, and a tool check with no check of another kind beside it on the module. +func healthProblems(m Manifest) []string { + var problems []string + endpoints := map[string]Listening{} + for _, l := range m.Listens { + if n := strings.TrimSpace(l.Name); n != "" { + endpoints[n] = l + } + } + tools := map[string]bool{} + for _, t := range m.Tools { + tools[t] = true + } + wants := map[string]bool{} + for _, w := range m.Wants() { + wants[w] = true + } + var toolChecks []string + otherKinds := 0 + for _, r := range m.Resources { + h, has, read := ReadHealth(r) + if !has { + continue + } + id, typ := fmt.Sprint(r["id"]), fmt.Sprint(r["type"]) + where := m.Module + ": " + id + for _, p := range read { + problems = append(problems, m.Module+": "+p) + } + if !LongRunning(r) { + problems = append(problems, fmt.Sprintf("%s declares health and does not stay up: a step, anything on a "+ + "schedule and a service not stated running are judged by their step and their schedule (ADR 0240 rule 1)", where)) + continue + } + switch h.Kind { + case HealthRuntime, HealthExec: + if typ != "container" { + problems = append(problems, fmt.Sprintf("%s is a %s and its health is %q, which only a container has: "+ + "the runtime runs it inside the container", where, typ, h.Kind)) + } + case HealthUnit: + if typ == "container" { + problems = append(problems, fmt.Sprintf("%s is a container and its health is %q, which is a service's "+ + "or a process's own readiness", where, h.Kind)) + } + case HealthHTTP, HealthTCP, HealthTool: + case "": + problems = append(problems, fmt.Sprintf("%s declares health with no kind: %s", where, healthKindsWords())) + default: + problems = append(problems, fmt.Sprintf("%s declares health of kind %q: %s", where, h.Kind, healthKindsWords())) + } + switch h.Kind { + case HealthHTTP, HealthTCP: + l, declared := endpoints[h.Endpoint] + switch { + case h.Endpoint == "": + problems = append(problems, fmt.Sprintf("%s's %s check names no endpoint: it names one the module "+ + "declares under listens, by its name", where, h.Kind)) + case !declared: + problems = append(problems, fmt.Sprintf("%s's %s check names the endpoint %q, which %s does not declare "+ + "under listens (%s)", where, h.Kind, h.Endpoint, m.Module, namedEndpointsWords(endpoints))) + case l.At() != "tcp": + problems = append(problems, fmt.Sprintf("%s's %s check names %q, which is %s: a check connects over tcp", + where, h.Kind, h.Endpoint, l.At())) + } + default: + if h.Endpoint != "" { + problems = append(problems, fmt.Sprintf("%s's %s check names an endpoint, which only an http or tcp check "+ + "looks at", where, h.Kind)) + } + } + if h.Kind != HealthHTTP && (h.Path != "" || h.Status != 0 || h.Body != "" || h.Scheme != "") { + problems = append(problems, fmt.Sprintf("%s's %s check says a path, a status, a body or a scheme, which "+ + "only an http check has", where, h.Kind)) + } + if h.Kind == HealthHTTP { + if h.Path != "" && !strings.HasPrefix(h.Path, "/") { + problems = append(problems, fmt.Sprintf("%s's http check asks %q; a path starts with /", where, h.Path)) + } + if h.Status != 0 && (h.Status < 100 || h.Status > 599) { + problems = append(problems, fmt.Sprintf("%s's http check expects status %d, which is not one", where, h.Status)) + } + if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" { + problems = append(problems, fmt.Sprintf("%s's http check is over %q; it is http or https", where, h.Scheme)) + } + } + if (h.Command != "") != (h.Kind == HealthExec) { + if h.Kind == HealthExec { + problems = append(problems, fmt.Sprintf("%s's exec check says no command", where)) + } else { + problems = append(problems, fmt.Sprintf("%s's %s check says a command, which only an exec check runs", + where, h.Kind)) + } + } + if h.Kind == HealthTool { + switch { + case h.Tool == "": + problems = append(problems, fmt.Sprintf("%s's tool check names no tool", where)) + case !tools[h.Tool]: + problems = append(problems, fmt.Sprintf("%s's tool check asks %q, which %s does not serve (its tools: %s)", + where, h.Tool, m.Module, orNoneWords(m.Tools))) + } + toolChecks = append(toolChecks, id) + } else { + if h.Tool != "" { + problems = append(problems, fmt.Sprintf("%s's %s check names a tool, which only a tool check asks", where, h.Kind)) + } + if h.Kind != "" { + otherKinds++ + } + } + if h.Needs != "" && !wants[h.Needs] { + problems = append(problems, fmt.Sprintf("%s's check needs %q, which %s does not require: a check names "+ + "the provision it exercises, among those the module requires", where, h.Needs, m.Module)) + } + problems = append(problems, healthTimingProblems(where, h)...) + } + if len(toolChecks) > 0 && otherKinds == 0 { + problems = append(problems, fmt.Sprintf("%s judges itself only by its own tool (%s): a tool check is for "+ + "function no endpoint shows, and only beside a check of another kind the module does not run itself "+ + "(ADR 0227 rule 8, ADR 0240 rule 2)", m.Module, strings.Join(toolChecks, ", "))) + } + return problems +} + +// healthTimingProblems holds a check's timing to its bounds. +func healthTimingProblems(where string, h Health) []string { + var problems []string + if h.Interval < HealthIntervalFloor { + problems = append(problems, fmt.Sprintf("%s looks every %s; a check looks no more often than every %s — "+ + "the mesh is a guest on the machine (ADR 0240 rule 2)", where, h.Interval, HealthIntervalFloor)) + } + if h.Timeout <= 0 || h.Timeout >= h.Interval { + problems = append(problems, fmt.Sprintf("%s gives a look %s, which must be more than nothing and under its "+ + "interval of %s", where, h.Timeout, h.Interval)) + } + if h.Looks < HealthLooksFloor { + problems = append(problems, fmt.Sprintf("%s is unhealthy after %d failing look(s); it is at least %d — one "+ + "look can be wrong (issue 277)", where, h.Looks, HealthLooksFloor)) + } + if h.Grace < 0 { + problems = append(problems, fmt.Sprintf("%s has a grace of %s", where, h.Grace)) + } + if h.Looks >= HealthLooksFloor && h.Interval >= HealthIntervalFloor { + if took := h.Grace + time.Duration(h.Looks)*h.Interval; took > HealthWithin { + problems = append(problems, fmt.Sprintf("%s is said unhealthy at the earliest %s after it starts (a grace "+ + "of %s and %d looks every %s); it is at most %s, so a resource broken from its start is said within "+ + "the gate's bound", where, took, h.Grace, h.Looks, h.Interval, HealthWithin)) + } + } + return problems +} + +func healthKindsWords() string { + return "a check is runtime (the image's own, adopted by name), http, tcp, exec, unit or tool" +} + +func namedEndpointsWords(endpoints map[string]Listening) string { + if len(endpoints) == 0 { + return "it names none" + } + names := make([]string, 0, len(endpoints)) + for n := range endpoints { + names = append(names, n) + } + sort.Strings(names) + return "it names " + strings.Join(names, ", ") +} + +func orNoneWords(names []string) string { + if len(names) == 0 { + return "none" + } + return strings.Join(names, ", ") +} + +// Undeclared is every long-running resource of the manifest without `health`, by id (ADR 0240 rule 8): +// what the catalogue's count counts. +func Undeclared(m Manifest) []string { + var out []string + for _, r := range m.Resources { + if !LongRunning(r) { + continue + } + if _, has := r[HealthField]; !has { + out = append(out, fmt.Sprint(r["id"])) + } + } + return out +} + +// HealthChecks is every tool a module's health asks, as `.`: what a machine's node-engine +// is granted to ask of its own node tools (to-be 48 §3). +func HealthChecks(m Manifest) []string { + var out []string + for _, r := range m.Resources { + h, has, _ := ReadHealth(r) + if has && h.Kind == HealthTool && h.Tool != "" && LongRunning(r) { + out = append(out, m.Module+"."+h.Tool) + } + } + sort.Strings(out) + return out +} + +// healthInto composes a resource's `health` into the node-engine's words, or takes it away (to-be 48 §2, +// §3): for an engine that reads it, the endpoint becomes the port this machine published it on and the +// defaults are written out; for one that does not — older and strict — the field is not sent, and the +// resource is judged by liveness alone, as before. +func healthInto(resource map[string]any, m Manifest, with Rendering) { + if _, has := resource[HealthField]; !has { + return + } + if !with.ReadsHealth { + delete(resource, HealthField) + return + } + h, _, _ := ReadHealth(resource) + out := map[string]any{"kind": h.Kind, "interval": h.Interval.String(), "timeout": h.Timeout.String(), + "looks": h.Looks, "grace": h.Grace.String()} + if h.Endpoint != "" { + out["endpoint"] = h.Endpoint + if port, ok := EndpointPort(m, h.Endpoint); ok { + out["port"] = with.machinePort(m.Module, port) + } + } + if h.Kind == HealthHTTP { + path := h.Path + if path == "" { + path = "/" + } + out["path"] = path + if h.Status != 0 { + out["status"] = h.Status + } + if h.Body != "" { + out["body"] = h.Body + } + if h.Scheme != "" { + out["scheme"] = h.Scheme + } + } + if h.Command != "" { + out["command"] = h.Command + } + if h.Tool != "" { + out["tool"] = h.Tool + } + if h.Needs != "" { + out["needs"] = h.Needs + } + resource[HealthField] = out +} + +// HealthWords is a declared check in a line, for `module check` and `node show`. +func HealthWords(h Health) string { + var what string + switch h.Kind { + case HealthHTTP: + what = "http " + orSlash(h.Path) + " on " + h.Endpoint + if h.Status != 0 { + what += " expecting " + strconv.Itoa(h.Status) + } + case HealthTCP: + what = "tcp on " + h.Endpoint + case HealthTool: + what = "its tool " + h.Tool + case HealthRuntime: + what = "its image's own check" + default: + what = h.Kind + } + return fmt.Sprintf("%s every %s", what, h.Interval) +} + +func orSlash(p string) string { + if p == "" { + return "/" + } + return p +} diff --git a/internal/catalogue/health_test.go b/internal/catalogue/health_test.go new file mode 100644 index 0000000..c5a68de --- /dev/null +++ b/internal/catalogue/health_test.go @@ -0,0 +1,217 @@ +package catalogue + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-host/validate" +) + +// A module says how each long-running resource is ready (novox/hq ADR 0240 rule 2, to-be 48 §8): `module +// check` refuses each part out of its bounds, each endpoint named by a port or an address, a tool the +// module does not serve, and a tool check alone — a test per refusal. + +const healthDigest = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +// healthManifest is a module of one web container, one running service and one step, with the health +// given on the container (or on the resource named by on). +func healthManifest(t *testing.T, on string, health map[string]any, more ...map[string]any) []byte { + t.Helper() + resources := []map[string]any{ + {"id": "server", "type": "container", "name": "app-server", "image": "registry.example/app" + healthDigest, + "ports": []any{"8080"}}, + {"id": "daemon", "type": "service", "unit": "app.service", "state": "running"}, + {"id": "seed", "type": "container", "name": "app-seed", "image": "registry.example/app" + healthDigest, + "run-once": true}, + {"id": "sweep", "type": "container", "name": "app-sweep", "image": "registry.example/app" + healthDigest, + "schedule": "0 3 * * *"}, + } + resources = append(resources, more...) + for _, r := range resources { + if r["id"] == on && health != nil { + r["health"] = health + } + } + m := map[string]any{"module": "app", "requires": []any{"postgres-database"}, "tools": []any{"app_status"}, + "listens": []any{ + map[string]any{"name": "web", "port": 8080, "from": "mesh"}, + map[string]any{"name": "beacon", "port": 9999, "protocol": "udp", "from": "mesh"}, + }, + "resources": resources} + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestAWellFormedHealthIsAccepted(t *testing.T) { + for _, h := range []map[string]any{ + {"kind": "http", "endpoint": "web", "path": "/healthz", "status": 200, "body": "ok", "needs": "postgres-database"}, + {"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "2s", "looks": 2, "grace": "0s"}, + {"kind": "runtime"}, + {"kind": "exec", "command": "pg_isready -q", "grace": "4m", "looks": 2, "interval": "30s"}, + } { + if _, err := ParseManifest(healthManifest(t, "server", h)); err != nil { + t.Errorf("%v was refused: %v", h, err) + } + } + if _, err := ParseManifest(healthManifest(t, "daemon", map[string]any{"kind": "unit"})); err != nil { + t.Errorf("a service's own readiness was refused: %v", err) + } + // A tool beside a check of another kind on the module. + raw := healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web"}, + map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running", + "health": map[string]any{"kind": "tool", "tool": "app_status"}}) + if _, err := ParseManifest(raw); err != nil { + t.Errorf("a tool check beside an http check was refused: %v", err) + } +} + +func TestEveryOutOfBoundsHealthIsRefusedByName(t *testing.T) { + cases := []struct { + name string + on string + health map[string]any + says string + }{ + {"no kind", "server", map[string]any{"endpoint": "web"}, "with no kind"}, + {"an unknown kind", "server", map[string]any{"kind": "ping"}, `of kind "ping"`}, + {"a port", "server", map[string]any{"kind": "tcp", "port": 8080}, "names a port"}, + {"an address", "server", map[string]any{"kind": "http", "endpoint": "web", "address": "127.0.0.1"}, "names a address"}, + {"a url", "server", map[string]any{"kind": "http", "url": "http://localhost:8080/"}, "names a url"}, + {"an unknown key", "server", map[string]any{"kind": "tcp", "endpoint": "web", "retries": 3}, `"retries"`}, + {"no endpoint", "server", map[string]any{"kind": "http"}, "names no endpoint"}, + {"an undeclared endpoint", "server", map[string]any{"kind": "tcp", "endpoint": "admin"}, "does not declare"}, + {"a udp endpoint", "server", map[string]any{"kind": "tcp", "endpoint": "beacon"}, "a check connects over tcp"}, + {"an interval under the floor", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "5s", "timeout": "1s"}, "no more often than every 10s"}, + {"a timeout of the interval", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "10s"}, "under its interval"}, + {"one failing look", "server", map[string]any{"kind": "tcp", "endpoint": "web", "looks": 1}, "at least 2"}, + {"a grace and looks past five minutes", "server", map[string]any{"kind": "tcp", "endpoint": "web", "grace": "4m", "looks": 3, "interval": "30s"}, "at most 5m0s"}, + {"a duration that is not one", "server", map[string]any{"kind": "tcp", "endpoint": "web", "interval": "often"}, "is a duration"}, + {"looks that are not a number", "server", map[string]any{"kind": "tcp", "endpoint": "web", "looks": "three"}, "whole number"}, + {"a path without a slash", "server", map[string]any{"kind": "http", "endpoint": "web", "path": "health"}, "starts with /"}, + {"a status that is not one", "server", map[string]any{"kind": "http", "endpoint": "web", "status": 700}, "is not one"}, + {"a scheme that is not one", "server", map[string]any{"kind": "http", "endpoint": "web", "scheme": "ftp"}, "http or https"}, + {"a status on a tcp check", "server", map[string]any{"kind": "tcp", "endpoint": "web", "status": 200}, "only an http check has"}, + {"an exec with no command", "server", map[string]any{"kind": "exec"}, "says no command"}, + {"a command on an http check", "server", map[string]any{"kind": "http", "endpoint": "web", "command": "true"}, "only an exec check runs"}, + {"a runtime check on a service", "daemon", map[string]any{"kind": "runtime"}, "only a container has"}, + {"a unit check on a container", "server", map[string]any{"kind": "unit"}, "a service's or a process's"}, + {"a tool the module does not serve", "server", map[string]any{"kind": "tool", "tool": "app_admin"}, "does not serve"}, + {"a tool check alone", "server", map[string]any{"kind": "tool", "tool": "app_status"}, "judges itself only by its own tool"}, + {"needs not required", "server", map[string]any{"kind": "tcp", "endpoint": "web", "needs": "redis"}, "does not require"}, + {"health on a step", "seed", map[string]any{"kind": "runtime"}, "does not stay up"}, + {"health on a schedule", "sweep", map[string]any{"kind": "runtime"}, "does not stay up"}, + {"health that is not an object", "server", nil, "is an object"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + raw := healthManifest(t, c.on, c.health) + if c.health == nil { + raw = []byte(strings.Replace(string(raw), `"name":"app-server"`, `"name":"app-server","health":"tcp"`, 1)) + } + _, err := ParseManifest(raw) + if err == nil { + t.Fatalf("%s was accepted", c.name) + } + if !strings.Contains(err.Error(), c.says) { + t.Errorf("refused, but not for saying %q: %v", c.says, err) + } + }) + } +} + +func TestHealthIsComposedAsThePortThisMachineGaveTheEndpoint(t *testing.T) { + m, err := ParseManifest(healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web", "path": "/healthz", + "needs": "postgres-database"})) + if err != nil { + t.Fatal(err) + } + compose := func(with Rendering) map[string]any { + out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with) + if err != nil { + t.Fatal(err) + } + at := indexOfID(out, "app.server") + if at < 0 { + t.Fatal("the container was lost") + } + return out[at] + } + // An engine older than the field is not sent it: it would refuse the whole declaration. + if h, sent := compose(Rendering{Ports: map[string]map[int]int{"app": {8080: 31001}}})["health"]; sent { + t.Fatalf("health was sent to an engine that does not read it: %v", h) + } + got := compose(Rendering{ReadsHealth: true, Ports: map[string]map[int]int{"app": {8080: 31001}}})["health"].(map[string]any) + if got["port"] != 31001 || got["endpoint"] != "web" || got["path"] != "/healthz" || got["needs"] != "postgres-database" { + t.Errorf("composed as %v", got) + } + if got["interval"] != "30s" || got["timeout"] != "5s" || got["looks"] != 3 || got["grace"] != "1m0s" { + t.Errorf("the defaults were not written out: %v", got) + } + // The port this machine gives the endpoint moves, and the check moves with it. + moved := compose(Rendering{ReadsHealth: true, Ports: map[string]map[int]int{"app": {8080: 31002}}})["health"].(map[string]any) + if moved["port"] != 31002 { + t.Errorf("after the port moved the check still dials %v", moved["port"]) + } + // And the catalogue's manifest is untouched by composing it. + if _, ok := m.Resources[0]["health"].(map[string]any)["port"]; ok { + t.Error("composing wrote the port into the catalogue's own manifest") + } +} + +func TestTheUndeclaredAreTheLongRunningWithoutHealth(t *testing.T) { + m, err := ParseManifest(healthManifest(t, "server", map[string]any{"kind": "runtime"})) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(Undeclared(m), ","); got != "daemon" { + t.Errorf("undeclared: %q; the step and the schedule are not long-running, the server declares", got) + } +} + +func TestATooledHealthIsGrantedToTheEngine(t *testing.T) { + raw := healthManifest(t, "server", map[string]any{"kind": "http", "endpoint": "web"}, + map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running", + "health": map[string]any{"kind": "tool", "tool": "app_status"}}) + m, err := ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + if got := HealthChecks(m); len(got) != 1 || got[0] != "app.app_status" { + t.Errorf("checks %v", got) + } +} + +// What the controller composes the node-engine takes: every kind, composed for an engine that reads it, +// passes the engine's own validator (mesh-host/validate) — one set of words on both sides. +func TestEveryComposedHealthIsOneTheNodeEngineTakes(t *testing.T) { + for _, h := range []map[string]any{ + {"kind": "http", "endpoint": "web", "path": "/healthz", "status": 200, "body": "ok", "needs": "postgres-database"}, + {"kind": "tcp", "endpoint": "web", "interval": "10s", "timeout": "2s", "looks": 2, "grace": "0s"}, + {"kind": "runtime"}, + {"kind": "exec", "command": "pg_isready -q"}, + } { + m, err := ParseManifest(healthManifest(t, "server", h, + map[string]any{"id": "admin", "type": "service", "unit": "app-admin.service", "state": "running", + "health": map[string]any{"kind": "tool", "tool": "app_status"}})) + if err != nil { + t.Fatal(err) + } + m.Resources[1]["health"] = map[string]any{"kind": "unit"} + out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(Rendering{ReadsHealth: true, + Ports: map[string]map[int]int{"app": {8080: 31001}}}) + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": out}) + if err != nil { + t.Fatal(err) + } + if problems := validate.Declaration(body); len(problems) > 0 { + t.Errorf("%v composed into something the node-engine refuses: %v", h, problems) + } + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index be03268..d13d272 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -2036,6 +2036,9 @@ func ParseManifest(raw []byte) (Manifest, error) { } problems = append(problems, zoneProblems(m)...) + // How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the + // words the node-engine would refuse it in far away. + problems = append(problems, healthProblems(m)...) if len(problems) > 0 { sort.Strings(problems) return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s", diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index 2b38bc9..8311527 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -137,6 +137,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio // And the state it keeps and reads (novox/hq ADR 0201). State: bucketsOf(m), Reads: m.Reads, + // And the tools its health asks (novox/hq ADR 0240): the machine's node-engine is granted them. + Checks: catalogue.HealthChecks(m), } // Whether it can be given an account at all: delivered as its own secret named broker, so one // that declares none has nowhere to read it (novox/hq issue 195). diff --git a/internal/inventory/health.go b/internal/inventory/health.go index 6d25898..a350080 100644 --- a/internal/inventory/health.go +++ b/internal/inventory/health.go @@ -24,6 +24,10 @@ type ResourceHealth struct { Since time.Time `json:"since"` Streak int `json:"streak,omitempty"` Restarts int `json:"restarts,omitempty"` + // Check is the declared check's kind, empty for liveness alone; Needs the provision it exercises + // (ADR 0240 Phase B, to-be 48 §6). + Check string `json:"check,omitempty"` + Needs string `json:"needs,omitempty"` } // NodeHealth is a machine's newest statement, as kept. diff --git a/internal/link/protocol.go b/internal/link/protocol.go index c56e6c6..e7767f7 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -415,6 +415,11 @@ func EnrolProof(secret string, public []byte, overlay, sealing, serving string) // LivenessContract is the version of the health statement this controller reads (ADR 0240 Phase A). const LivenessContract = 1 +// ReadinessContract is the statement of an engine that also reads a resource's declared `health` and +// judges it (ADR 0240 Phase B): only to an engine whose statement says this or later is the field sent, +// because an older one parses strictly and would refuse the whole declaration for it. +const ReadinessContract = 2 + // Health is one statement of a machine's long-running resources (to-be 48 §4): in every report, as the // event HealthSubject between reports on each change, and again every minute while one is not healthy. // The node-engine's own (mesh-host internal/link Health); a test on each side holds the field names. @@ -445,6 +450,10 @@ type ResourceHealth struct { Since time.Time `json:"since"` Streak int `json:"streak,omitempty"` Restarts int `json:"restarts,omitempty"` + // Check is the declared check's kind (ADR 0240 Phase B), empty for a resource judged by liveness + // alone; Needs is the provision the check exercises (to-be 48 §6). + Check string `json:"check,omitempty"` + Needs string `json:"needs,omitempty"` } // HealthSaid is the health event's body: the machine and its statement. The machine is read from the diff --git a/internal/link/protocol_test.go b/internal/link/protocol_test.go index 7c1e89c..4f8a412 100644 --- a/internal/link/protocol_test.go +++ b/internal/link/protocol_test.go @@ -29,8 +29,9 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { {Report{Node: "n", Health: &Health{Contract: LivenessContract}}, []string{"node", "health"}}, {Health{Contract: LivenessContract, Resources: []ResourceHealth{}}, []string{"contract", "at", "resources"}}, {ResourceHealth{Module: "m", Resource: "m.r", Kind: "container", Target: "t", State: StateUnhealthy, - Reason: "restarting", Streak: 2, Restarts: 3}, - []string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts"}}, + Reason: "restarting", Streak: 2, Restarts: 3, Check: "http", Needs: "postgres-database"}, + []string{"module", "resource", "kind", "target", "state", "reason", "since", "streak", "restarts", + "check", "needs"}}, {HealthSaid{Node: "n"}, []string{"node", "health"}}, } { raw, err := json.Marshal(c.value) diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index dc908cb..2472adf 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -590,6 +590,23 @@ type Process struct { // For a process that stays up; a step or a scheduled run is not running a moment later by // design, so there is nothing to hand over to. Replaces []string `json:"replaces,omitempty"` + + // Witness is how the node-engine judges a new build of this process, and restores the build + // before it when the new one is not healthy in bound (novox/hq to-be 45 §8): "lease" — the + // controller this machine started holds the controller's lease; "ping" — this machine's runtime + // answers the services protocol's PING; "none". Absent is the default for the process's name: + // the mesh's two core processes are judged, nothing else is. For a process that stays up. + Witness string `json:"witness,omitempty"` + + // NotReversible says why this build may not be rolled back, when it may not: the build before it + // would run against what this one changes — a migration it runs that the older build cannot read + // (to-be 45 §8, rule 8). A build so declared that is not healthy in bound is left running and said + // as urgent; the build before it is never started against the newer data. + NotReversible string `json:"not-reversible,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` } func (d *Process) Identity() string { return d.ID } @@ -621,7 +638,7 @@ func ProcessNameProblem(name string) string { } func (d *Process) validate(where string, _ bool) []string { - var problems []string + problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "") if problem := ProcessNameProblem(d.Name); problem != "" { problems = append(problems, where+": "+problem) } @@ -637,6 +654,19 @@ func (d *Process) validate(where string, _ bool) []string { if len(d.Run) == 0 { problems = append(problems, where+": a process needs to say what to run") } + switch d.Witness { + case "", "lease", "ping", "none": + default: + problems = append(problems, fmt.Sprintf("%s: witness %q is not one this host keeps: lease, ping or none", + where, d.Witness)) + } + if d.Witness != "" && d.Witness != "none" && (d.RunOnce || d.Schedule != "") { + problems = append(problems, where+": a witness judges a process that stays up; a step or a "+ + "scheduled run is not running between its runs") + } + if strings.ContainsAny(d.NotReversible, "\n\r") { + problems = append(problems, where+": not-reversible is one line") + } for _, part := range d.Run { if part == "" { problems = append(problems, where+": a process command has an empty element") @@ -756,6 +786,10 @@ type Service struct { // said by the controller, which knows the found tunnel's key is this node's own: without that, // starting this unit on the found one's port would drop every peer's packets. TakesOver *TakeOver `json:"takes-over,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` } // TakeOver is a found tunnel a service replaces: its interface, the unit that raised it, and its @@ -784,7 +818,7 @@ const ( func (s *Service) UserScoped() bool { return s.Scope == ScopeUser } func (s *Service) validate(where string, _ bool) []string { - var problems []string + problems := s.Health.problems(where, false, s.State == "running") if s.Unit == "" { problems = append(problems, where+": a service needs a unit") } @@ -1096,6 +1130,10 @@ type Container struct { // offline job says *before*, not *instead of*; a recurring window is the case order cannot // express, and the only one this serves. WhileStopped []string `json:"while-stopped,omitempty"` + + // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its + // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. + Health *Health `json:"health,omitempty"` } func (c *Container) Identity() string { return c.ID } @@ -1103,7 +1141,7 @@ func (c *Container) Kind() Type { return TypeContainer } func (c *Container) Target() string { return c.Name } func (c *Container) validate(where string, _ bool) []string { - var problems []string + problems := c.Health.problems(where, true, !c.RunOnce && c.Schedule == "") if c.Name == "" { problems = append(problems, where+": a container needs a name") } diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/health.go b/vendor/github.com/novox/mesh-host/internal/declaration/health.go new file mode 100644 index 0000000..d6395bf --- /dev/null +++ b/vendor/github.com/novox/mesh-host/internal/declaration/health.go @@ -0,0 +1,188 @@ +package declaration + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "time" +) + +// Health is how a long-running resource is ready, as the controller composed it from the module's +// `health` (novox/hq ADR 0240 rule 2, to-be 48 §2–§3, Phase B): one kind and its timing, the endpoint +// already the port this machine published it on. +// +// **The node-engine runs every kind and owns every verdict.** http and tcp it makes itself, from the +// machine to the port; unit it reads from the service manager it already reads; exec and runtime it hands +// to the container runtime as the container's own check, with this timing, and reads the state; tool it +// asks of its own node tools. Nothing else on the machine sets a container's check. +// +// Refused here as the controller refuses it near the author, in the same bounds: an engine that took a +// check it could not judge would say a module ready that nothing looked at. +type Health struct { + Kind string `json:"kind"` + // Endpoint is the module's name for what Port is: for the words a verdict is said in. + Endpoint string `json:"endpoint,omitempty"` + Port int `json:"port,omitempty"` + Path string `json:"path,omitempty"` + Status int `json:"status,omitempty"` + Body string `json:"body,omitempty"` + Scheme string `json:"scheme,omitempty"` + Command string `json:"command,omitempty"` + Tool string `json:"tool,omitempty"` + Interval string `json:"interval"` + Timeout string `json:"timeout"` + Looks int `json:"looks"` + Grace string `json:"grace"` + // Needs is the provision the check exercises (to-be 48 §6): said with every verdict, so the + // controller can hold what it finds under the provider's own condition. + Needs string `json:"needs,omitempty"` +} + +// UnmarshalJSON reads a health strictly, as everything a declaration carries is read: a field this host +// does not know is a part of the check the controller believes it asked for, and nothing would look at it. +func (h *Health) UnmarshalJSON(raw []byte) error { + type plain Health + var p plain + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return fmt.Errorf("health: %w", err) + } + *h = Health(p) + return nil +} + +// The kinds. +const ( + HealthRuntime = "runtime" + HealthHTTP = "http" + HealthTCP = "tcp" + HealthExec = "exec" + HealthUnit = "unit" + HealthTool = "tool" +) + +// The bounds (ADR 0240 rule 2) — the controller's, held again here. +const ( + HealthIntervalFloor = 10 * time.Second + HealthLooksFloor = 2 + HealthWithin = 5 * time.Minute +) + +// Every, Within and GraceOf are the timing, read. Validated on arrival, so a parse error here is +// impossible on a declaration that was accepted; it reads as zero. +func (h *Health) Every() time.Duration { d, _ := time.ParseDuration(h.Interval); return d } +func (h *Health) Within() time.Duration { d, _ := time.ParseDuration(h.Timeout); return d } +func (h *Health) GraceOf() time.Duration { d, _ := time.ParseDuration(h.Grace); return d } + +// RunByRuntime says the container runtime runs this check as the container's own: exec and runtime. +func (h *Health) RunByRuntime() bool { + return h != nil && (h.Kind == HealthExec || h.Kind == HealthRuntime) +} + +// Words is the check in a few words, as a verdict is said: "http /healthz on web". +func (h *Health) Words() string { + switch h.Kind { + case HealthHTTP: + return "http " + h.Path + " on " + orPort(h.Endpoint, h.Port) + case HealthTCP: + return "tcp on " + orPort(h.Endpoint, h.Port) + case HealthTool: + return "its tool " + h.Tool + case HealthRuntime: + return "its image's own check" + case HealthExec: + return "its command" + case HealthUnit: + return "its unit" + } + return h.Kind +} + +func orPort(endpoint string, port int) string { + if endpoint != "" { + return endpoint + } + return fmt.Sprint(port) +} + +// problems holds a resource's health to its kind and bounds. container says whether the resource is a +// container; longRunning whether it stays up. +func (h *Health) problems(where string, container, longRunning bool) []string { + if h == nil { + return nil + } + var problems []string + say := func(format string, args ...any) { + problems = append(problems, where+": "+fmt.Sprintf(format, args...)) + } + if !longRunning { + say("health is judged on what stays up; a step or a scheduled run is judged by its own outcome") + } + switch h.Kind { + case HealthHTTP, HealthTCP: + if h.Port < 1 || h.Port > 65535 { + say("a %s check needs the port it looks at", h.Kind) + } + case HealthExec: + if !container { + say("an exec check runs inside a container") + } + if strings.TrimSpace(h.Command) == "" { + say("an exec check needs a command") + } + case HealthRuntime: + if !container { + say("a runtime check is a container image's own") + } + case HealthUnit: + if container { + say("a unit check is a service's or a process's own") + } + case HealthTool: + if strings.TrimSpace(h.Tool) == "" { + say("a tool check names the tool") + } + default: + say("health of kind %q; it is runtime, http, tcp, exec, unit or tool", h.Kind) + } + if h.Kind == HealthHTTP { + if !strings.HasPrefix(h.Path, "/") { + say("an http check asks a path starting with /") + } + if h.Status != 0 && (h.Status < 100 || h.Status > 599) { + say("an http check expects status %d, which is not one", h.Status) + } + if h.Scheme != "" && h.Scheme != "http" && h.Scheme != "https" { + say("an http check is over http or https, not %q", h.Scheme) + } + } + if strings.ContainsAny(h.Command, "\n\r") { + say("an exec check's command is one line") + } + every, everyErr := time.ParseDuration(h.Interval) + within, withinErr := time.ParseDuration(h.Timeout) + grace, graceErr := time.ParseDuration(h.Grace) + switch { + case everyErr != nil || withinErr != nil || graceErr != nil: + say("health's interval, timeout and grace are durations") + default: + if every < HealthIntervalFloor { + say("a check looks no more often than every %s, not every %s", HealthIntervalFloor, every) + } + if within <= 0 || within >= every { + say("a look takes more than nothing and less than its interval") + } + if grace < 0 { + say("a grace is not negative") + } + if h.Looks >= HealthLooksFloor && grace+time.Duration(h.Looks)*every > HealthWithin { + say("a grace and the failing looks take at most %s", HealthWithin) + } + } + if h.Looks < HealthLooksFloor { + say("a check is unhealthy after at least %d failing looks, not %d", HealthLooksFloor, h.Looks) + } + return problems +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ef2f5e3..6e2738b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -75,7 +75,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/validate @@ -133,4 +133,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261006095519-3e80b7ae325e +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261007120832-bdd44154ccac