Read how a module says each resource is ready, and send it to engines that read it (hq ADR 0240, to-be 48 Phase B)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check fail: its merge-check.sh failed: FAIL github.com/novox/mesh-controller/cmd/mesh-controller 1800.059s
mesh/delivery-group group feat/health-the-field rejected: its order contradicts itself: novox/mesh-controller@468d509462fe, novox/mesh-host@96bf9415aae4
mesh/delivery superseded: a newer head of the same pull request

A module could say nothing about what ready means for what it runs, so a web
application with its port open and its requests hanging passed everything for
eleven hours (issue 145). A long-running resource now carries `health` — the
image's own check adopted by name, http, tcp, exec, unit or a module's own tool,
with its timing — refused near its author when it names a port or an address,
an endpoint the module does not declare, a tool it does not serve, a tool check
alone, or a timing outside the record's bounds. It is composed with the endpoint
as the port this machine published it on, and sent only to a node-engine whose
statement says it reads it: an older one would refuse the whole declaration.
The engine is granted its own machine's instance of each health tool. `module
check` warns of every long-running resource without `health`, counts them for
the catalogue, and refuses them from 2026-11-18. A check's findings stay out of
a condition's summary. The node-engine's validator is vendored at its Phase B
commit, so what is composed is judged by the words the engine takes.
This commit is contained in:
jochen
2026-10-07 14:39:13 +02:00
parent 863ebd4277
commit 468d509462
22 changed files with 1205 additions and 10 deletions
+46
View File
@@ -8,6 +8,7 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
@@ -107,6 +108,28 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
names = append(names, name)
}
sort.Strings(names)
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
// change lower it, never raise it.
undeclared := 0
required := !checkNow().Before(catalogue.HealthRequiredFrom)
for _, name := range names {
missing := catalogue.Undeclared(shelf[name])
undeclared += len(missing)
if len(missing) == 0 {
continue
}
if required {
for _, id := range missing {
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
}
failed += len(missing)
faulted[name] = true
}
}
for _, name := range names {
m := shelf[name]
if faulted[name] {
@@ -138,8 +161,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
}
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
}
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
var checks []string
for _, r := range m.Resources {
if h, has, _ := catalogue.ReadHealth(r); has {
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
}
}
if len(checks) > 0 {
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
}
if missing := catalogue.Undeclared(m); len(missing) > 0 {
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
}
fmt.Fprintln(out)
}
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
if failed > 0 {
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
}
@@ -150,6 +189,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
return nil
}
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
const UndeclaredHealthLine = "long-running resources without health:"
// checkNow is the clock `module check` judges the date by; a test sets it.
var checkNow = time.Now
func joinInvokes(invokes []string) string {
if len(invokes) == 1 && invokes[0] == "*" {
return "every tool"
+47
View File
@@ -0,0 +1,47 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
dir := t.TempDir()
digest := "@sha256:" + strings.Repeat("a", 64)
path := filepath.Join(dir, "module.json")
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
"health":{"kind":"http","endpoint":"web"}},
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
defer func() { checkNow = time.Now }()
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
var out bytes.Buffer
if err := moduleCheck([]string{path}, &out); err != nil {
t.Fatalf("refused before the date: %v\n%s", err, out.String())
}
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
if !strings.Contains(out.String(), want) {
t.Errorf("the check does not say %q:\n%s", want, out.String())
}
}
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
out.Reset()
if err := moduleCheck([]string{path}, &out); err == nil {
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
}
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
t.Errorf("the refusal does not name the resource:\n%s", out.String())
}
}
+7 -1
View File
@@ -71,7 +71,8 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
for _, r := range h.Resources {
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts}
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
Check: r.Check, Needs: r.Needs}
resources = append(resources, kept)
if r.State == link.StateUnhealthy && r.Module != "" {
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
@@ -172,6 +173,11 @@ func reasonWords(r inventory.ResourceHealth) string {
case "":
return "is unhealthy"
}
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
// operator's channel carries (ADR 0234 §6). The summary names the check.
if r.Check != "" {
return "fails its " + r.Check + " check"
}
return "is unhealthy: " + r.Reason
}
+33
View File
@@ -161,3 +161,36 @@ func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
}
}
// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
reads := func() bool {
t.Helper()
got, err := engineReadsHealth(ctx, inv, "anchor")
if err != nil {
t.Fatal(err)
}
return got
}
if reads() {
t.Fatal("an engine that never stated anything is sent health")
}
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
t.Fatal(err)
}
if reads() {
t.Fatal("an engine judging liveness alone is sent health")
}
later := aStatement(h0.Add(time.Minute), link.StateHealthy)
later.Contract = link.ReadinessContract
if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if !reads() {
t.Fatal("an engine that reads health is not sent it")
}
}
+17
View File
@@ -17,6 +17,7 @@ import (
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/licences"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/overlay"
)
@@ -860,7 +861,12 @@ func renderingFor(ctx context.Context, open *stores, node string,
if artifactStore != "" {
reach = map[string]string{"mesh-artifact-store": artifactStore}
}
readsHealth, err := engineReadsHealth(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
ReadsHealth: readsHealth,
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
@@ -872,6 +878,17 @@ func renderingFor(ctx context.Context, open *stores, node string,
}, record, nil
}
// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240
// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the
// field, because it parses strictly and would refuse the whole declaration for it.
func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.ReadinessContract, nil
}
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
// 0199): the zone settled from that node's settings, the node's private address, the port the
// answering listen is published on there.
+55
View File
@@ -10,6 +10,7 @@ import (
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -286,3 +287,57 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
}
}
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its
// program ran while every request hung, and the mesh said its machine was healthy; a person found it.
// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check
// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it
// found looking at such a program; here the controller hears that statement on two looks in a row and
// raises the module's condition — the second, never the first. `null` is an engine older than the
// readiness check: it states the program alive, and nothing is raised.
func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
`"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` +
`"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`)
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("the engine's statement of the silent web application: %v", err)
}
silent = raw
}
var h link.Health
if err := json.Unmarshal(silent, &h); err != nil {
t.Fatal(err)
}
if h.Contract == 0 {
t.Fatal("the engine states nothing of the silent web application: it is older than the judging")
}
open1 := func() []conditions.Condition {
t.Helper()
list, err := conditionsFrom.Open(ctx)
if err != nil {
t.Fatal(err)
}
return list
}
for look := 1; look <= 2; look++ {
at := time.Now().Add(time.Duration(look) * time.Second)
h.At = at
if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil {
t.Fatal(err)
}
raised := open1()
switch {
case look == 1 && len(raised) != 0:
t.Fatalf("one look raised %v", raised[0].Key)
case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"):
t.Fatalf("two looks in a row did not raise the module's condition: %v", raised)
case look == 2:
t.Logf("raised on the second look: %s", raised[0].Summary)
}
}
}