Read how a module says each resource is ready, and send it to engines that read it (hq ADR 0240, to-be 48 Phase B)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group feat/health-the-field delivering: 1 of 3 delivered
mesh/delivery held for a person: merged, and the controller opened no walk for it within 10m0s — nothing it holds follows that branch, or the merge was…
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery-group group feat/health-the-field delivering: 1 of 3 delivered
mesh/delivery held for a person: merged, and the controller opened no walk for it within 10m0s — nothing it holds follows that branch, or the merge was…
A module could say nothing about what ready means for what it runs, so a web application with its port open and its requests hanging passed everything for eleven hours (issue 145). A long-running resource now carries `health` — the image's own check adopted by name, http, tcp, exec, unit or a module's own tool, with its timing — refused near its author when it names a port or an address, an endpoint the module does not declare, a tool it does not serve, a tool check alone, or a timing outside the record's bounds. It is composed with the endpoint as the port this machine published it on, and sent only to a node-engine whose statement says it reads it: an older one would refuse the whole declaration. The engine is granted its own machine's instance of each health tool. `module check` warns of every long-running resource without `health`, counts them for the catalogue, and refuses them from 2026-11-18. A check's findings stay out of a condition's summary. The node-engine's validator is vendored at its Phase B commit, so what is composed is judged by the words the engine takes.
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
@@ -107,6 +108,28 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
||||
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
||||
// change lower it, never raise it.
|
||||
undeclared := 0
|
||||
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
||||
for _, name := range names {
|
||||
missing := catalogue.Undeclared(shelf[name])
|
||||
undeclared += len(missing)
|
||||
if len(missing) == 0 {
|
||||
continue
|
||||
}
|
||||
if required {
|
||||
for _, id := range missing {
|
||||
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
||||
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
failed += len(missing)
|
||||
faulted[name] = true
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
@@ -138,8 +161,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
||||
}
|
||||
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
||||
var checks []string
|
||||
for _, r := range m.Resources {
|
||||
if h, has, _ := catalogue.ReadHealth(r); has {
|
||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
||||
}
|
||||
}
|
||||
if len(checks) > 0 {
|
||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
||||
}
|
||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
@@ -150,6 +189,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
||||
const UndeclaredHealthLine = "long-running resources without health:"
|
||||
|
||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
||||
var checkNow = time.Now
|
||||
|
||||
func joinInvokes(invokes []string) string {
|
||||
if len(invokes) == 1 && invokes[0] == "*" {
|
||||
return "every tool"
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
|
||||
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
|
||||
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
digest := "@sha256:" + strings.Repeat("a", 64)
|
||||
path := filepath.Join(dir, "module.json")
|
||||
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
|
||||
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
|
||||
"health":{"kind":"http","endpoint":"web"}},
|
||||
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
|
||||
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
|
||||
defer func() { checkNow = time.Now }()
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
||||
t.Fatalf("refused before the date: %v\n%s", err, out.String())
|
||||
}
|
||||
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
|
||||
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
|
||||
out.Reset()
|
||||
if err := moduleCheck([]string{path}, &out); err == nil {
|
||||
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
|
||||
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -71,7 +71,8 @@ func stateHealth(ctx context.Context, inv *inventory.Inventory, k *conditions.Ke
|
||||
resources := make([]inventory.ResourceHealth, 0, len(h.Resources))
|
||||
for _, r := range h.Resources {
|
||||
kept := inventory.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target,
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts}
|
||||
State: r.State, Reason: r.Reason, Since: r.Since, Streak: r.Streak, Restarts: r.Restarts,
|
||||
Check: r.Check, Needs: r.Needs}
|
||||
resources = append(resources, kept)
|
||||
if r.State == link.StateUnhealthy && r.Module != "" {
|
||||
unhealthy[r.Module] = append(unhealthy[r.Module], kept)
|
||||
@@ -172,6 +173,11 @@ func reasonWords(r inventory.ResourceHealth) string {
|
||||
case "":
|
||||
return "is unhealthy"
|
||||
}
|
||||
// What a declared check found says an endpoint, a path or an address: evidence, never the summary the
|
||||
// operator's channel carries (ADR 0234 §6). The summary names the check.
|
||||
if r.Check != "" {
|
||||
return "fails its " + r.Check + " check"
|
||||
}
|
||||
return "is unhealthy: " + r.Reason
|
||||
}
|
||||
|
||||
|
||||
@@ -161,3 +161,36 @@ func TestAReportWithNoHealthRaisesAndKeepsNothing(t *testing.T) {
|
||||
t.Fatalf("the report's health was not kept: %+v %v %v", kept, had, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A declared `health` is sent only to an engine whose own statement says it reads it (novox/hq ADR 0240
|
||||
// Phase B): an older engine is strict and would refuse the whole declaration for the field.
|
||||
func TestHealthIsSentOnlyToAnEngineThatSaysItReadsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
reads := func() bool {
|
||||
t.Helper()
|
||||
got, err := engineReadsHealth(ctx, inv, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine that never stated anything is sent health")
|
||||
}
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", aStatement(h0, link.StateHealthy), h0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reads() {
|
||||
t.Fatal("an engine judging liveness alone is sent health")
|
||||
}
|
||||
later := aStatement(h0.Add(time.Minute), link.StateHealthy)
|
||||
later.Contract = link.ReadinessContract
|
||||
if err := stateHealth(ctx, inv, nil, "anchor", later, h0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reads() {
|
||||
t.Fatal("an engine that reads health is not sent it")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
@@ -860,7 +861,12 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if artifactStore != "" {
|
||||
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||
}
|
||||
readsHealth, err := engineReadsHealth(ctx, inv, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
ReadsHealth: readsHealth,
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
@@ -872,6 +878,17 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
// engineReadsHealth says whether a machine's node-engine reads a declared `health` (novox/hq ADR 0240
|
||||
// Phase B), by its own newest statement: one older, or one that never stated anything, is not sent the
|
||||
// field, because it parses strictly and would refuse the whole declaration for it.
|
||||
func engineReadsHealth(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
|
||||
stated, had, err := inv.HealthOf(ctx, node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return had && stated.Contract >= link.ReadinessContract, nil
|
||||
}
|
||||
|
||||
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
|
||||
// 0199): the zone settled from that node's settings, the node's private address, the port the
|
||||
// answering listen is published on there.
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -286,3 +287,57 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq
|
||||
// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its
|
||||
// program ran while every request hung, and the mesh said its machine was healthy; a person found it.
|
||||
// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check
|
||||
// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it
|
||||
// found looking at such a program; here the controller hears that statement on two looks in a row and
|
||||
// raises the module's condition — the second, never the first. `null` is an engine older than the
|
||||
// readiness check: it states the program alive, and nothing is raised.
|
||||
func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` +
|
||||
`"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` +
|
||||
`"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`)
|
||||
if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("the engine's statement of the silent web application: %v", err)
|
||||
}
|
||||
silent = raw
|
||||
}
|
||||
var h link.Health
|
||||
if err := json.Unmarshal(silent, &h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.Contract == 0 {
|
||||
t.Fatal("the engine states nothing of the silent web application: it is older than the judging")
|
||||
}
|
||||
open1 := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
list, err := conditionsFrom.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return list
|
||||
}
|
||||
for look := 1; look <= 2; look++ {
|
||||
at := time.Now().Add(time.Duration(look) * time.Second)
|
||||
h.At = at
|
||||
if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raised := open1()
|
||||
switch {
|
||||
case look == 1 && len(raised) != 0:
|
||||
t.Fatalf("one look raised %v", raised[0].Key)
|
||||
case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"):
|
||||
t.Fatalf("two looks in a row did not raise the module's condition: %v", raised)
|
||||
case look == 2:
|
||||
t.Logf("raised on the second look: %s", raised[0].Summary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user