Keep mesh-mail-reader's own health, surviving a restart (issue 1519)

2026-10-20 23:58:37 +02:00
parent d7b5c87887
commit 183cc58347
6 changed files with 194 additions and 16 deletions
@@ -0,0 +1,49 @@
+package main
+
+// mesh-mail-reader's own health (novox/hq ADR 0459 §12, issue 1519): a tool the module's runtime-health of
+// kind tool asks (mesh-vault's own pattern, module.json:68-75, novox/hq ADR 0366 §2 — a one-process module may
+// judge itself by its own tool; the context file for this milestone says mesh-mail-notifier's health.go STOP
+// comment, which says otherwise, is out of date). Unhealthy once a batch judgement is refused, times out or
+// answers out of shape, naming which of the three, the judgement's own words, and since when (the first
+// failure's own time, local); healthy again once a judgement succeeds. The state survives a restart: it is
+// kept in the module's own database (store.go, pgstore.go), not only in memory.
+
+import (
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// HealthTool is the module's own health tool (the manifest's runtime-health, and one of its seven tools).
+const HealthTool = "mail_reader_health"
+
+// RecordJudgement folds one batch judgement's outcome into the kept health state (1519/AC2, AC3): a refused,
+// timed-out or out-of-shape judgement makes it unhealthy, naming the cause and the judgement's own words, and
+// keeps the FIRST failure's own since — a second, third, ... failure in a row never pushes since forward. A
+// judgement that succeeds makes it healthy again at once.
+func RecordJudgement(current ReaderHealth, outcome HealthOutcome, why string, now time.Time) ReaderHealth {
+	if outcome == OutcomeOK {
+		return ReaderHealth{Healthy: true}
+	}
+	since := now
+	if !current.Healthy && !current.Since.IsZero() {
+		since = current.Since
+	}
+	return ReaderHealth{Healthy: false, Cause: outcome, Why: why, Since: since}
+}
+
+// Answer is the state as mail_reader_health answers it (1519/AC2, AC3): unhealthy names which of the three
+// outcomes, the judgement's own words, and since when, in local; healthy otherwise.
+func (h ReaderHealth) Answer(local *time.Location) health.Answer {
+	if h.Healthy {
+		return health.Healthy("the last batch judgement succeeded")
+	}
+	since := h.Since
+	if local != nil {
+		since = since.In(local)
+	} else {
+		since = since.Local()
+	}
+	return health.Unhealthy(fmt.Sprintf("a batch judgement was %s: %s (since %s)", h.Cause, h.Why, since.Format("15:04 2006-01-02")))
+}
@@ -0,0 +1,109 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+// 1519/AC1: the manifest declares runtime-health {kind: tool, tool: mail_reader_health}.
+func TestTheManifestDeclaresRuntimeHealth(t *testing.T) {
+	raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var m struct {
+		RuntimeHealth struct {
+			Kind string `json:"kind"`
+			Tool string `json:"tool"`
+		} `json:"runtime-health"`
+	}
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatal(err)
+	}
+	if m.RuntimeHealth.Kind != "tool" || m.RuntimeHealth.Tool != HealthTool {
+		t.Fatalf("runtime-health = %+v, want kind tool, tool %s", m.RuntimeHealth, HealthTool)
+	}
+}
+
+// 1519/AC2: after a judgement refused, timed out or answered out of shape, mail_reader_health answers
+// unhealthy with why naming which of the three, the judgement's own words, and since when; later failures
+// keep the first since.
+func TestRecordJudgementUnhealthyKeepsTheFirstSince(t *testing.T) {
+	first := time.Date(2026, 10, 20, 9, 30, 0, 0, time.UTC)
+	after := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeRefused, "holds no licence", first)
+	if after.Healthy || after.Cause != OutcomeRefused || after.Why != "holds no licence" || !after.Since.Equal(first) {
+		t.Fatalf("after one refusal: %+v", after)
+	}
+
+	second := first.Add(5 * time.Minute)
+	again := RecordJudgement(after, OutcomeTimedOut, "no answer in 40s", second)
+	if again.Healthy || again.Cause != OutcomeTimedOut || again.Why != "no answer in 40s" {
+		t.Fatalf("after a second, different failure: %+v", again)
+	}
+	if !again.Since.Equal(first) {
+		t.Fatalf("since = %v, want the FIRST failure's own time %v, not the second's %v", again.Since, first, second)
+	}
+
+	a := again.Answer(time.UTC)
+	if a.Healthy || !strings.Contains(a.Why, "timed out") || !strings.Contains(a.Why, "no answer in 40s") ||
+		!strings.Contains(a.Why, "09:30") {
+		t.Fatalf("Answer().Why = %q, want it to name the cause, the judgement's own words, and since 09:30", a.Why)
+	}
+}
+
+// 1519/AC3: after a judgement that succeeds, it answers healthy.
+func TestRecordJudgementSuccessIsHealthyAgain(t *testing.T) {
+	unhealthy := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeOutOfShape, "not an object", time.Now())
+	healthy := RecordJudgement(unhealthy, OutcomeOK, "", time.Now())
+	if !healthy.Healthy || healthy.Cause != "" || !healthy.Since.IsZero() {
+		t.Fatalf("after a success: %+v", healthy)
+	}
+	a := healthy.Answer(time.UTC)
+	if a.State != health.StateHealthy {
+		t.Fatalf("Answer() = %+v, want healthy", a)
+	}
+}
+
+// 1519/AC4: the state is kept in the module's database, so a restart answers the same — proven here with the
+// store's in-memory double (kit_test.go); the SQL itself (reader_health) is pgstore_test.go's own concern.
+func TestHealthToolReadsTheKeptState(t *testing.T) {
+	store := newMemStore()
+	unhealthy := RecordJudgement(ReaderHealth{Healthy: true}, OutcomeRefused, "holds no licence", time.Now())
+	if err := store.SetReaderHealth(context.Background(), unhealthy); err != nil {
+		t.Fatal(err)
+	}
+
+	// A fresh Bundle, as a restarted process would build one, reading the same store.
+	b := &Bundle{
+		Caller:          func(map[string]any) string { return "mesh-mail-web" },
+		callersOverride: func() (Callers, error) { return Callers{"mesh-mail-web"}, nil },
+		Store:           func(context.Context) (Store, error) { return store, nil },
+	}
+	got, err := b.healthTool(map[string]any{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	a, ok := got.(health.Answer)
+	if !ok || a.Healthy {
+		t.Fatalf("mail_reader_health after a restart = %+v, want the kept unhealthy state", got)
+	}
+}
+
+// 1519: reader_health's own table exists in the schema, with a column for each of the kept state's parts.
+func TestTheSchemaHasAReaderHealthTable(t *testing.T) {
+	if !strings.Contains(consolidated, "create table reader_health") {
+		t.Fatal("schema.sql declares no reader_health table")
+	}
+	for _, column := range []string{"healthy", "cause", "why", "since"} {
+		if !strings.Contains(consolidated, column) {
+			t.Fatalf("schema.sql's reader_health has no column %q", column)
+		}
+	}
+}
@@ -3,7 +3,7 @@
 // mesh-mail runs on. It hears mesh-mail's added events under its own runs-as account (ADR 0459 §3, issue
 // 1518), judges a batch of them at a time (1427, 1520 — not yet built), and serves the six mail-reading
 // verbs (entries, groups, correct, lessons, undo, status) and its own health tool, mail_reader_health
-// (1519 — not yet built). It reads mail; it moves, flags, deletes and sends nothing (novox/hq ADR 0459 §11).
+// (1519). It reads mail; it moves, flags, deletes and sends nothing (novox/hq ADR 0459 §11).
 //
 //	mesh-mail-reader            the bundle the node's runtime launches: the hearing loop (hear.go) and the
 //	                            six mail-reading verbs plus mail_reader_health, served over stdio (tools.go)
@@ -36,7 +36,7 @@ func main() {
 		os.Exit(prepare())
 	}
 	storeFor := func(ctx context.Context) (Store, error) { return PgStoreFromEnv(ctx) }
-	b := &Bundle{Env: envFromOS()}
+	b := &Bundle{Env: envFromOS(), Store: storeFor}
 	go hearLoop(storeFor, stdio.Subscribe, time.Sleep)
 	if err := stdio.ServeOn("", b.Tools(), os.Stdin, os.Stdout); err != nil {
 		logf("[mesh-mail-reader] %v", err)
@@ -7,11 +7,13 @@ package main
 // asked by the node-engine itself and carries no caller check.
 //
 // The six verbs' own behaviour — reading and writing mail entries, mail groups and reading lessons — is
-// 1521 (entries, groups, status) and 1428 (correct, lessons, undo); mail_reader_health's own answer is 1519.
-// Until each lands, its tool answers that it is not built yet, after the caller check has already passed or
-// refused, so a caller never learns whether a verb is built before learning whether it is granted.
+// 1521 (entries, groups, status) and 1428 (correct, lessons, undo). Until each lands, its tool answers that
+// it is not built yet, after the caller check has already passed or refused, so a caller never learns
+// whether a verb is built before learning whether it is granted. mail_reader_health's own answer is built
+// (1519): it reaches the kept health state (store.go, health.go) instead.

 import (
+	"context"
 	"errors"
 	"fmt"

@@ -29,16 +31,13 @@ const (
 	StatusTool  = "mail-reading.status"
 )

-// HealthTool is the module's own health tool (1519): the manifest's runtime-health, and the seventh of its
-// seven tools. Defined here, not in a health.go of its own yet, so Tools() below compiles before 1519 lands.
-const HealthTool = "mail_reader_health"
-
-// Bundle is what the tools reach: the trusted setting `callers`, and the runtime's own caller. Caller is nil
-// outside a test (CallerOf is read from the runtime). The index (store.go) is 1517; the health tool reaches it
-// from 1519 on.
+// Bundle is what the tools reach: the trusted setting `callers`, the runtime's own caller, and the index
+// (store.go), which mail_reader_health reads (1519). Caller is nil outside a test (CallerOf is read from the
+// runtime).
 type Bundle struct {
 	Env    Env
 	Caller func(args map[string]any) string
+	Store  func(ctx context.Context) (Store, error)

 	// callersOverride, when set, is used instead of reading Env.Callers — the tests' own seam.
 	callersOverride func() (Callers, error)
@@ -235,5 +234,14 @@ func (b *Bundle) healthTool(args map[string]any) (any, error) {
 	if err := only(args); err != nil {
 		return nil, err
 	}
-	return nil, notBuiltYet(HealthTool, "1519")
+	ctx := context.Background()
+	store, err := b.Store(ctx)
+	if err != nil {
+		return nil, err
+	}
+	h, err := store.ReaderHealth(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return h.Answer(nil), nil
 }
@@ -1,6 +1,7 @@
 package main

 import (
+	"context"
 	"strings"
 	"testing"
 )
@@ -86,10 +87,13 @@ func TestAMailReadingVerbRefusesWhenCallersCannotBeRead(t *testing.T) {

 // mail_reader_health carries no caller check (1426: "mail_reader_health is asked by the node-engine itself").
 func TestHealthToolHasNoCallerCheck(t *testing.T) {
-	b := &Bundle{callersOverride: func() (Callers, error) { return nil, errFake }}
+	b := &Bundle{
+		callersOverride: func() (Callers, error) { return nil, errFake },
+		Store:           func(context.Context) (Store, error) { return newMemStore(), nil },
+	}
 	_, err := b.healthTool(map[string]any{})
-	if err == nil || strings.Contains(err.Error(), "refused") {
-		t.Fatalf("mail_reader_health: %v, want it to skip the caller check entirely", err)
+	if err != nil {
+		t.Fatalf("mail_reader_health: %v, want it to skip the caller check entirely and answer", err)
 	}
 }

@@ -45,6 +45,14 @@
    "mail-reading.status",
    "mail_reader_health"
  ],
+  "runtime-health": {
+    "kind": "tool",
+    "tool": "mail_reader_health",
+    "interval": "60s",
+    "timeout": "10s",
+    "looks": 2,
+    "grace": "60s"
+  },
  "own-secrets": {
    "broker": "${dir:state}/broker"
  },