mesh-healers writes its findings through mesh-sdk's finding writer and the generated finding

Builds: design 47 G §686, §689, A-3, A-13, A-14, A-16

Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
2026-11-11 21:49:17 +01:00
parent 9677f200f3
commit b78e90c0c7
6 changed files with 233 additions and 72 deletions
@@ -0,0 +1,96 @@
package main
// The healers' findings are the generated finding of the seat mesh-conditions, written through mesh-sdk's finding writer
// (design 47 G §461, §689).
import (
"context"
"encoding/json"
"fmt"
"sort"
"time"
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
// Finding, and the writer's other three bodies, are the seat mesh-conditions' own types.
type (
Finding = meshconditions.Finding
FindingCleared = meshconditions.FindingCleared
FindingsStated = meshconditions.FindingsStated
FindingsAsked = meshconditions.FindingsAskedEvent
FindingsWriter = views.Writer[Finding, FindingCleared, FindingsStated, FindingsAsked]
)
// finding is f about scope and id: it sets the key's scope and id and the subject's scope and id equal (design 47 G A-14).
func finding(scope, id string, f Finding) Finding {
f.Scope, f.ID = scope, keys.EscapeName(id)
f.Subject = meshconditions.Subject{Scope: scope, ID: f.ID}
return f
}
// evidence is the one observation a finding keeps as evidence; its moment is the finding's own, so that a finding that says
// the same is not written again.
func evidence(s string) []meshconditions.Evidence { return []meshconditions.Evidence{{Said: s}} }
// keyOf is where f is kept: `<scope>.<id>.<kind>`.
func keyOf(f Finding) string { return views.FindingKey(f.Scope, f.ID, f.Kind) }
// PendingBound is how long a finding the healers cannot judge yet stays open (design 47 G §384).
const PendingBound = 5 * time.Minute
// stored are the findings in the state `findings`, by key.
func (h *Healers) stored() (map[string]Finding, error) {
keys, err := h.Store.Keys(FindingsState)
if err != nil {
return nil, err
}
out := map[string]Finding{}
for _, k := range keys {
raw, found, err := h.Store.Get(FindingsState, k)
if err != nil {
return nil, err
}
if !found {
continue
}
var f Finding
if json.Unmarshal(raw, &f) == nil {
out[k] = f
}
}
return out, nil
}
// Observe is the writer's observe (design 47 G §384): every finding the healers observe now, judged from what they
// keep, and the stored findings they cannot judge yet.
func (h *Healers) Observe(ctx context.Context) (views.Observed[Finding], error) {
stored, err := h.stored()
if err != nil {
return views.Observed[Finding]{}, fmt.Errorf("its own findings cannot be read: %w", err)
}
now := h.Now()
var out views.Observed[Finding]
pend := func(source string) {
for k, f := range stored {
if f.Source == source {
out.Pending = append(out.Pending, views.PendingKey{Key: k, Since: f.Since, Bound: PendingBound.String()})
}
}
}
if all, err := h.attempts(); err != nil {
out.Findings = append(out.Findings, blindFinding(err))
pend("brake")
} else if braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now); braked {
out.Findings = append(out.Findings, brakeFinding(actsWithin(all, now.Add(-BrakeWindow)), said))
}
if wanted, _, seeded := h.wantedNow(now); seeded {
out.Findings = append(out.Findings, wanted...)
} else {
pend("S15")
}
sort.Slice(out.Pending, func(i, j int) bool { return out.Pending[i].Key < out.Pending[j].Key })
return out, nil
}
@@ -14,6 +14,7 @@ package main
// bucket's age); nothing writes there any more.
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -29,6 +30,7 @@ import (
handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
)
// Module and Seat are this module and the seat it holds once for the mesh.
@@ -181,12 +183,19 @@ type Healers struct {
Now func() time.Time
Logf func(format string, args ...any)
Registry []Healer
// Writer is the finding writer over this module's state `findings` (views.NewStdioWriter; design 47 G A-16), whose
// observe is Observe.
Writer *FindingsWriter
mu sync.Mutex
// declined is why each condition was last passed over, said once.
declined map[string]string
// acts are the hand-act log as watched, for S15.
acts map[string]HandAct
// actsMu guards acts and seeded, which the writer's observe reads while it holds the writer's lock; it is never held
// while a finding is written.
actsMu sync.Mutex
// acts are the hand-act log as watched, for S15; seeded is whether the seat's answer at the start was taken.
acts map[string]HandAct
seeded bool
}
// actingWhy is why the healers act: what the tool `healers` says.
@@ -240,7 +249,7 @@ func (h *Healers) Judge(c Condition, by string) (string, error) {
h.see("blind", nil)
acts := actsWithin(all, now.Add(-BrakeWindow))
if braked, said := h.brakeHolds(acts, now); braked {
h.see("brake", []meshconditionsclient.Observation{brakeFinding(acts, said)})
h.see("brake", []Finding{brakeFinding(acts, said)})
return "every healer has stopped: " + said, nil
}
h.see("brake", nil)
@@ -483,10 +492,11 @@ func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) {
return false, said
}
var brakeKey = meshconditionsclient.Key(meshconditionsclient.ScopeMesh, "healers", "braked")
// brakeKey is where the brake's finding is kept.
var brakeKey = views.FindingKey(meshconditionsclient.ScopeMesh, "healers", "healers-braked")
// brakeFinding is the brake, as the urgent finding that says it.
func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation {
func brakeFinding(acts []Attempt, said string) Finding {
counts := map[string]int{}
for _, a := range acts {
counts[a.Healer+" on "+a.Condition]++
@@ -496,67 +506,53 @@ func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation
most = append(most, fmt.Sprintf("%s ×%d", what, n))
}
sort.Strings(most)
return meshconditionsclient.Observation{Kind: "healers-braked", Ref: "braked", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeMesh, ID: "healers",
return finding(meshconditionsclient.ScopeMesh, "healers", Finding{Kind: "healers-braked", Severity: meshconditionsclient.Urgent,
Source: "brake",
Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " +
"healer looping, not repairing — " + said,
Said: strings.Join(most, ", "),
Evidence: evidence(strings.Join(most, ", ")),
Headline: "All automatic repairs stopped",
Needs: "read the details and decide whether the repairs may run again.",
Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " +
"like a loop, so all of them stopped.",
Resolved: "Automatic repairs run again"}
"like a loop, so all of them stopped."})
}
// blindFinding is the healers' own blindness: they cannot read what they did.
func blindFinding(err error) Finding {
return finding(meshconditionsclient.ScopeProbe, "healers", Finding{Kind: "probe-failed", Severity: meshconditionsclient.Warning,
Source: "blind",
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Evidence: evidence(firstLine(err.Error())), Headline: "A check of the mesh could not run",
Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can."})
}
// blind writes the healers' own blindness: they cannot read what they did.
func (h *Healers) blind(err error) {
h.see("blind", []meshconditionsclient.Observation{{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeProbe, ID: "healers",
Summary: "the healers cannot read what they did, so they count no budget and act on nothing",
Said: firstLine(err.Error()), Headline: "A check of the mesh could not run",
Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.",
Resolved: "The automatic repairs read what they did again"}})
}
func (h *Healers) blind(err error) { h.see("blind", []Finding{blindFinding(err)}) }
// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it
// no longer sees (ADR 0564 §2).
func (h *Healers) see(source string, found []meshconditionsclient.Observation) {
now := h.Now()
want := map[string]meshconditions.FindingsByKeyValue{}
for _, o := range found {
o.Source = source
want[o.Key()] = o.Finding(Module, "", now)
// see writes the healers' own findings of one source through the finding writer, each only when it changes, and clears
// those of the source it no longer sees (ADR 0564 §2; design 47 G §461).
func (h *Healers) see(source string, found []Finding) {
ctx := context.Background()
want := map[string]Finding{}
for _, f := range found {
f.Source = source
want[keyOf(f)] = f
}
keys, err := h.Store.Keys(FindingsState)
stored, err := h.stored()
if err != nil {
h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err)
return
}
for _, k := range keys {
raw, found, err := h.Store.Get(FindingsState, k)
if err != nil || !found {
for k, was := range stored {
if _, still := want[k]; still || was.Source != source {
continue
}
var was meshconditions.FindingsByKeyValue
err = json.Unmarshal(raw, &was)
if err != nil || was.Source != source {
continue
}
if f, still := want[k]; still {
if meshconditionsclient.Says(was) == meshconditionsclient.Says(f) {
delete(want, k)
}
continue
}
if err := h.Store.Delete(FindingsState, k); err != nil {
h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err)
if err := h.Writer.Clear(ctx, was.Scope, was.ID, was.Kind, "the healers no longer see it"); err != nil {
h.Logf("[%s] the finding %s no longer seen is not cleared: %v", Module, k, err)
}
}
for k, f := range want {
raw, err := json.Marshal(f)
if err == nil {
err = h.Store.Put(FindingsState, k, json.RawMessage(raw))
}
if err != nil {
if err := h.Writer.Write(ctx, f); err != nil {
h.Logf("[%s] the finding %s is not written: %v", Module, k, err)
}
}
@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -15,6 +16,8 @@ import (
meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
)
type memStore struct {
@@ -131,6 +134,8 @@ type world struct {
now time.Time
asked []string
said []Acted
// events are the subjects the finding writer said its events on.
events []string
}
func newWorld() *world {
@@ -155,8 +160,35 @@ func (w *world) fresh() {
return nil
},
}
wr, err := views.NewWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{
Module: Module, Node: "ace", State: findingState{w.store}, Observe: w.h.Observe, Now: func() time.Time { return w.now },
Publish: func(_ context.Context, subject string, _ []byte, _ string) error {
w.events = append(w.events, subject)
return nil
},
Listen: func(context.Context, string, func([]byte)) (func(), error) { return func() {}, nil }})
if err != nil {
panic(err)
}
w.h.Writer = wr
}
// findingState is the writer's view of the store's state `findings`.
type findingState struct{ s *memStore }
func (f findingState) Get(key string) (*stdio.StateEntry, error) {
raw, ok, err := f.s.Get(FindingsState, key)
if err != nil || !ok {
return nil, err
}
return &stdio.StateEntry{Key: key, Value: raw}, nil
}
func (f findingState) Put(key string, value any) (uint64, error) {
return 0, f.s.Put(FindingsState, key, value)
}
func (f findingState) Delete(key string) error { return f.s.Delete(FindingsState, key) }
func (f findingState) Keys() ([]string, error) { return f.s.Keys(FindingsState) }
// drill is a healer of the tests alone: no healer acts here since H3 ended with the controller (issue 1126), and the
// budget, the settle, the escalation and the brake are every healer's, judged on it.
var drill = Healer{ID: "HD", Kinds: []string{"drill-broken"}, Condition: "a drill", Repair: "ask the drill again",
@@ -288,11 +320,18 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
t.Errorf("past the brake: %q, asked %v", said, b.asked)
}
raw, braked, _ := b.store.Get(FindingsState, brakeKey)
var f meshconditions.FindingsByKeyValue
var f Finding
_ = json.Unmarshal(raw, &f)
if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module {
if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module || f.Since.IsZero() {
t.Errorf("the brake is written as %s", raw)
}
if !slices.Contains(b.events, "mesh.stream.mesh-healers.event.finding-written.ace") {
t.Errorf("the brake was not said as finding-written on the writer's subject: %v", b.events)
}
// Judged anew at a start, from what the healers keep: the brake still holds, and states the same finding.
if obs, err := b.h.Observe(context.Background()); err != nil || len(obs.Findings) != 1 || obs.Findings[0].Kind != "healers-braked" {
t.Errorf("a statement of the brake: %+v, %v", obs, err)
}
b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held
if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") {
t.Errorf("the brake let go before an hour after the last act: %q", said)
@@ -310,7 +349,7 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 {
t.Errorf("blind, the healers acted: %v", err)
}
if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) {
if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.probe-failed"}) {
t.Errorf("blind, the findings are %v", keys)
}
}
@@ -381,7 +420,7 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) {
"mesh-vault.secret_vault_export"} {
act(fmt.Sprintf("n%d", i), verb, "operator", time.Hour)
}
want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"}
want := []string{"mesh.hand-acts=2econsumer-behind.healer-wanted"}
if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) {
t.Fatalf("findings %v, want %v", keys, want)
}
@@ -7,6 +7,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -14,6 +15,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
@@ -55,6 +57,13 @@ func main() {
h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf,
Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
Publish: stdio.SeatPublish}
w, err := views.NewStdioWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{
Module: Module, State: stdio.State(FindingsState), Observe: h.Observe, Logf: logf})
if err != nil {
logf("%v", err)
os.Exit(1)
}
h.Writer = w
// At start, a settle that ended while no instance watched is judged (Rearm); the waits still running are keys on
// the bus, heard on their expiry.
go h.Rearm()
@@ -66,6 +75,9 @@ func main() {
// durable consumer (the controller's bucket is no longer read, issue 774).
go func() {
<-stdio.Served()
if err := h.Writer.Start(context.Background()); err != nil {
logf("[%s] the finding writer did not start: %v", Module, err)
}
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
answer, err := stdio.Ask("seat:mesh-hand-acts.hand-acts", map[string]any{"days": int(Within.Hours() / 24)})
if err == nil {
+29 -21
View File
@@ -82,12 +82,12 @@ func (h *Healers) ActRecorded(body []byte) {
if json.Unmarshal(body, &a) != nil || a.ID == "" {
return
}
h.mu.Lock()
h.actsMu.Lock()
if h.acts == nil {
h.acts = map[string]HandAct{}
}
h.acts[handacts.Seat+":"+a.ID] = a
h.mu.Unlock()
h.actsMu.Unlock()
h.Wanted()
}
@@ -106,6 +106,9 @@ func (h *Healers) Seeded(answer []byte) (int, error) {
for _, raw := range l.Acts {
h.ActRecorded(raw)
}
h.actsMu.Lock()
h.seeded = true
h.actsMu.Unlock()
// A partial answer (the seat bounds it, issue 1966) seeds all the same: the newest acts are what a fortnight's
// count needs most, and the older ones are heard no more, so asking again would only repeat it.
if l.Complete != nil && !*l.Complete {
@@ -115,11 +118,25 @@ func (h *Healers) Seeded(answer []byte) (int, error) {
}
// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more,
// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight.
func (h *Healers) Wanted() []meshconditionsclient.Observation {
h.mu.Lock()
defer h.mu.Unlock()
// clearing the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight.
func (h *Healers) Wanted() []Finding {
now := h.Now()
found, next, _ := h.wantedNow(now)
h.see("S15", found)
if !next.IsZero() {
ttl := next.Sub(now).Round(time.Second) + time.Second
if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil {
h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err)
}
}
return found
}
// wantedNow is the finding of every cause repaired by hand twice or more within the fortnight, and when the oldest act
// of a wanted cause leaves it; seeded says the seat's acts of the start were taken, so that none is judged before.
func (h *Healers) wantedNow(now time.Time) (found []Finding, next time.Time, seeded bool) {
h.actsMu.Lock()
defer h.actsMu.Unlock()
byCause := map[string][]HandAct{}
for _, a := range h.acts {
if personsDecision(a) || now.Sub(a.At) > Within {
@@ -134,8 +151,6 @@ func (h *Healers) Wanted() []meshconditionsclient.Observation {
}
}
sort.Strings(causes)
var found []meshconditionsclient.Observation
var next time.Time
for _, cause := range causes {
acts := byCause[cause]
sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
@@ -153,27 +168,20 @@ func (h *Healers) Wanted() []meshconditionsclient.Observation {
}
}
words := strings.NewReplacer(".", " ", "_", " ").Replace(cause)
found = append(found, meshconditionsclient.Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: meshconditionsclient.Warning,
Scope: meshconditionsclient.ScopeMesh, ID: "hand-acts." + cause,
f := finding(meshconditionsclient.ScopeMesh, "hand-acts."+cause, Finding{Kind: "healer-wanted", Severity: meshconditionsclient.Warning,
Source: "S15",
Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts),
int(Within.Hours()/24), newest.By, wanted),
Said: strings.Join(lines, "; "),
Evidence: evidence(strings.Join(lines, "; ")),
Headline: capital(words) + " keeps being fixed by hand",
Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+
"for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)),
Resolved: "Resolved: no more hand repairs of " + words})
"for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24))})
found = append(found, f)
if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) {
next = leaves
}
}
h.see("S15", found)
if !next.IsZero() {
ttl := next.Sub(now).Round(time.Second) + time.Second
if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil {
h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err)
}
}
return found
return found, next, h.seeded
}
func capital(s string) string {
@@ -30,6 +30,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/testbus"
"git.novox.be/novox/mesh-sdk/go/views"
)
type wire struct {
@@ -360,6 +361,10 @@ func TestTheHealersWholeHearTheKeeperAndPassOverWhatEnded(t *testing.T) {
if err != nil {
t.Fatal(err)
}
stated, err := nc.SubscribeSync("mesh.mod." + Module + ".event.findings-stated")
if err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module)
stdin, _ := cmd.StdinPipe()
@@ -384,6 +389,11 @@ func TestTheHealersWholeHearTheKeeperAndPassOverWhatEnded(t *testing.T) {
// The runtime's consumer holds an event until the bundle subscribes: delivered once it hears them.
eventually(t, "the healers hearing the keeper's conditions", func() bool { return p.hears(RaisedEvent) && p.hears(ChangedEvent) })
// The writer hears the keeper's asks for findings, and states at its start the findings it observes: none.
eventually(t, "the writer hearing findings-asked", func() bool { return p.hears(views.FindingsAskedEvent) })
if msg, err := stated.NextMsg(10 * time.Second); err != nil || !strings.Contains(string(msg.Data), `"findings":[]`) {
t.Errorf("findings-stated at the start: %v", err)
}
// What ended with the controller is passed over: no act begun, nothing asked, nothing said.
p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records"))
if msg, err := acted.NextMsg(3 * time.Second); err == nil {