From 183cc5834773886c601f4527d6246a2814316bda Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 20 Oct 2026 23:58:37 +0200 Subject: [PATCH] Keep mesh-mail-reader's own health, surviving a restart (issue 1519) --- .../cmd/mesh-mail-reader/health.go | 49 ++++++++ .../cmd/mesh-mail-reader/health_test.go | 109 ++++++++++++++++++ .../cmd/mesh-mail-reader/main.go | 4 +- .../cmd/mesh-mail-reader/tools.go | 30 +++-- .../cmd/mesh-mail-reader/tools_test.go | 10 +- modules/mesh-mail-reader/module.json | 8 ++ 6 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-mail-reader/cmd/mesh-mail-reader/health.go create mode 100644 modules/mesh-mail-reader/cmd/mesh-mail-reader/health_test.go diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/health.go b/modules/mesh-mail-reader/cmd/mesh-mail-reader/health.go new file mode 100644 index 00000000..78c97064 --- /dev/null +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/health.go @@ -0,0 +1,49 @@ +package main + +// mesh-mail-reader's own health (novox/hq ADR 0459 §12, issue 1519): a tool the module's runtime-health of +// kind tool asks (mesh-vault's own pattern, module.json:68-75, novox/hq ADR 0366 §2 — a one-process module may +// judge itself by its own tool; the context file for this milestone says mesh-mail-notifier's health.go STOP +// comment, which says otherwise, is out of date). Unhealthy once a batch judgement is refused, times out or +// answers out of shape, naming which of the three, the judgement's own words, and since when (the first +// failure's own time, local); healthy again once a judgement succeeds. The state survives a restart: it is +// kept in the module's own database (store.go, pgstore.go), not only in memory. + +import ( + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// HealthTool is the module's own health tool (the manifest's runtime-health, and one of its seven tools). +const HealthTool = "mail_reader_health" + +// RecordJudgement folds one batch judgement's outcome into the kept health state (1519/AC2, AC3): a refused, +// timed-out or out-of-shape judgement makes it unhealthy, naming the cause and the judgement's own words, and +// keeps the FIRST failure's own since — a second, third, ... failure in a row never pushes since forward. A +// judgement that succeeds makes it healthy again at once. +func RecordJudgement(current ReaderHealth, outcome HealthOutcome, why string, now time.Time) ReaderHealth { + if outcome == OutcomeOK { + return ReaderHealth{Healthy: true} + } + since := now + if !current.Healthy && !current.Since.IsZero() { + since = current.Since + } + return ReaderHealth{Healthy: false, Cause: outcome, Why: why, Since: since} +} + +// Answer is the state as mail_reader_health answers it (1519/AC2, AC3): unhealthy names which of the three +// outcomes, the judgement's own words, and since when, in local; healthy otherwise. +func (h ReaderHealth) Answer(local *time.Location) health.Answer { + if h.Healthy { + return health.Healthy("the last batch judgement succeeded") + } + since := h.Since + if local != nil { + since = since.In(local) + } else { + since = since.Local() + } + return health.Unhealthy(fmt.Sprintf("a batch judgement was %s: %s (since %s)", h.Cause, h.Why, since.Format("15:04 2006-01-02"))) +} diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/health_test.go b/modules/mesh-mail-reader/cmd/mesh-mail-reader/health_test.go new file mode 100644 index 00000000..bc0a71e1 --- /dev/null +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/health_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// 1519/AC1: the manifest declares runtime-health {kind: tool, tool: mail_reader_health}. +func TestTheManifestDeclaresRuntimeHealth(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + RuntimeHealth struct { + Kind string `json:"kind"` + Tool string `json:"tool"` + } `json:"runtime-health"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != HealthTool { + t.Fatalf("runtime-health = %+v, want kind tool, tool %s", m.RuntimeHealth, HealthTool) + } +} + +// 1519/AC2: after a judgement refused, timed out or answered out of shape, mail_reader_health answers +// unhealthy with why naming which of the three, the judgement's own words, and since when; later failures +// keep the first since. +func TestRecordJudgementUnhealthyKeepsTheFirstSince(t *testing.T) { + first := time.Date(2026, 10, 20, 9, 30, 0, 0, time.UTC) + after := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeRefused, "holds no licence", first) + if after.Healthy || after.Cause != OutcomeRefused || after.Why != "holds no licence" || !after.Since.Equal(first) { + t.Fatalf("after one refusal: %+v", after) + } + + second := first.Add(5 * time.Minute) + again := RecordJudgement(after, OutcomeTimedOut, "no answer in 40s", second) + if again.Healthy || again.Cause != OutcomeTimedOut || again.Why != "no answer in 40s" { + t.Fatalf("after a second, different failure: %+v", again) + } + if !again.Since.Equal(first) { + t.Fatalf("since = %v, want the FIRST failure's own time %v, not the second's %v", again.Since, first, second) + } + + a := again.Answer(time.UTC) + if a.Healthy || !strings.Contains(a.Why, "timed out") || !strings.Contains(a.Why, "no answer in 40s") || + !strings.Contains(a.Why, "09:30") { + t.Fatalf("Answer().Why = %q, want it to name the cause, the judgement's own words, and since 09:30", a.Why) + } +} + +// 1519/AC3: after a judgement that succeeds, it answers healthy. +func TestRecordJudgementSuccessIsHealthyAgain(t *testing.T) { + unhealthy := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeOutOfShape, "not an object", time.Now()) + healthy := RecordJudgement(unhealthy, OutcomeOK, "", time.Now()) + if !healthy.Healthy || healthy.Cause != "" || !healthy.Since.IsZero() { + t.Fatalf("after a success: %+v", healthy) + } + a := healthy.Answer(time.UTC) + if a.State != health.StateHealthy { + t.Fatalf("Answer() = %+v, want healthy", a) + } +} + +// 1519/AC4: the state is kept in the module's database, so a restart answers the same — proven here with the +// store's in-memory double (kit_test.go); the SQL itself (reader_health) is pgstore_test.go's own concern. +func TestHealthToolReadsTheKeptState(t *testing.T) { + store := newMemStore() + unhealthy := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeRefused, "holds no licence", time.Now()) + if err := store.SetReaderHealth(context.Background(), unhealthy); err != nil { + t.Fatal(err) + } + + // A fresh Bundle, as a restarted process would build one, reading the same store. + b := &Bundle{ + Caller: func(map[string]any) string { return "mesh-mail-web" }, + callersOverride: func() (Callers, error) { return Callers{"mesh-mail-web"}, nil }, + Store: func(context.Context) (Store, error) { return store, nil }, + } + got, err := b.healthTool(map[string]any{}) + if err != nil { + t.Fatal(err) + } + a, ok := got.(health.Answer) + if !ok || a.Healthy { + t.Fatalf("mail_reader_health after a restart = %+v, want the kept unhealthy state", got) + } +} + +// 1519: reader_health's own table exists in the schema, with a column for each of the kept state's parts. +func TestTheSchemaHasAReaderHealthTable(t *testing.T) { + if !strings.Contains(consolidated, "create table reader_health") { + t.Fatal("schema.sql declares no reader_health table") + } + for _, column := range []string{"healthy", "cause", "why", "since"} { + if !strings.Contains(consolidated, column) { + t.Fatalf("schema.sql's reader_health has no column %q", column) + } + } +} diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/main.go b/modules/mesh-mail-reader/cmd/mesh-mail-reader/main.go index 883645ad..b323b196 100644 --- a/modules/mesh-mail-reader/cmd/mesh-mail-reader/main.go +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/main.go @@ -3,7 +3,7 @@ // mesh-mail runs on. It hears mesh-mail's added events under its own runs-as account (ADR 0459 §3, issue // 1518), judges a batch of them at a time (1427, 1520 — not yet built), and serves the six mail-reading // verbs (entries, groups, correct, lessons, undo, status) and its own health tool, mail_reader_health -// (1519 — not yet built). It reads mail; it moves, flags, deletes and sends nothing (novox/hq ADR 0459 §11). +// (1519). It reads mail; it moves, flags, deletes and sends nothing (novox/hq ADR 0459 §11). // // mesh-mail-reader the bundle the node's runtime launches: the hearing loop (hear.go) and the // six mail-reading verbs plus mail_reader_health, served over stdio (tools.go) @@ -36,7 +36,7 @@ func main() { os.Exit(prepare()) } storeFor := func(ctx context.Context) (Store, error) { return PgStoreFromEnv(ctx) } - b := &Bundle{Env: envFromOS()} + b := &Bundle{Env: envFromOS(), Store: storeFor} go hearLoop(storeFor, stdio.Subscribe, time.Sleep) if err := stdio.ServeOn("", b.Tools(), os.Stdin, os.Stdout); err != nil { logf("[mesh-mail-reader] %v", err) diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools.go b/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools.go index fd431899..e4546a04 100644 --- a/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools.go +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools.go @@ -7,11 +7,13 @@ package main // asked by the node-engine itself and carries no caller check. // // The six verbs' own behaviour — reading and writing mail entries, mail groups and reading lessons — is -// 1521 (entries, groups, status) and 1428 (correct, lessons, undo); mail_reader_health's own answer is 1519. -// Until each lands, its tool answers that it is not built yet, after the caller check has already passed or -// refused, so a caller never learns whether a verb is built before learning whether it is granted. +// 1521 (entries, groups, status) and 1428 (correct, lessons, undo). Until each lands, its tool answers that +// it is not built yet, after the caller check has already passed or refused, so a caller never learns +// whether a verb is built before learning whether it is granted. mail_reader_health's own answer is built +// (1519): it reaches the kept health state (store.go, health.go) instead. import ( + "context" "errors" "fmt" @@ -29,16 +31,13 @@ const ( StatusTool = "mail-reading.status" ) -// HealthTool is the module's own health tool (1519): the manifest's runtime-health, and the seventh of its -// seven tools. Defined here, not in a health.go of its own yet, so Tools() below compiles before 1519 lands. -const HealthTool = "mail_reader_health" - -// Bundle is what the tools reach: the trusted setting `callers`, and the runtime's own caller. Caller is nil -// outside a test (CallerOf is read from the runtime). The index (store.go) is 1517; the health tool reaches it -// from 1519 on. +// Bundle is what the tools reach: the trusted setting `callers`, the runtime's own caller, and the index +// (store.go), which mail_reader_health reads (1519). Caller is nil outside a test (CallerOf is read from the +// runtime). type Bundle struct { Env Env Caller func(args map[string]any) string + Store func(ctx context.Context) (Store, error) // callersOverride, when set, is used instead of reading Env.Callers — the tests' own seam. callersOverride func() (Callers, error) @@ -235,5 +234,14 @@ func (b *Bundle) healthTool(args map[string]any) (any, error) { if err := only(args); err != nil { return nil, err } - return nil, notBuiltYet(HealthTool, "1519") + ctx := context.Background() + store, err := b.Store(ctx) + if err != nil { + return nil, err + } + h, err := store.ReaderHealth(ctx) + if err != nil { + return nil, err + } + return h.Answer(nil), nil } diff --git a/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools_test.go b/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools_test.go index 9fc3c808..b088dde3 100644 --- a/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools_test.go +++ b/modules/mesh-mail-reader/cmd/mesh-mail-reader/tools_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "strings" "testing" ) @@ -86,10 +87,13 @@ func TestAMailReadingVerbRefusesWhenCallersCannotBeRead(t *testing.T) { // mail_reader_health carries no caller check (1426: "mail_reader_health is asked by the node-engine itself"). func TestHealthToolHasNoCallerCheck(t *testing.T) { - b := &Bundle{callersOverride: func() (Callers, error) { return nil, errFake }} + b := &Bundle{ + callersOverride: func() (Callers, error) { return nil, errFake }, + Store: func(context.Context) (Store, error) { return newMemStore(), nil }, + } _, err := b.healthTool(map[string]any{}) - if err == nil || strings.Contains(err.Error(), "refused") { - t.Fatalf("mail_reader_health: %v, want it to skip the caller check entirely", err) + if err != nil { + t.Fatalf("mail_reader_health: %v, want it to skip the caller check entirely and answer", err) } } diff --git a/modules/mesh-mail-reader/module.json b/modules/mesh-mail-reader/module.json index ff16ca7f..7e05017d 100644 --- a/modules/mesh-mail-reader/module.json +++ b/modules/mesh-mail-reader/module.json @@ -45,6 +45,14 @@ "mail-reading.status", "mail_reader_health" ], + "runtime-health": { + "kind": "tool", + "tool": "mail_reader_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, "own-secrets": { "broker": "${dir:state}/broker" },