diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md index 0431657c..b266558b 100644 --- a/modules/mesh-healers/README.md +++ b/modules/mesh-healers/README.md @@ -18,6 +18,12 @@ mid-act has spent it and a second instance cannot begin it again. The condition **Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. +Both buckets age as a whole, each key a week after its last write: the runtime gives a bundle no time to live per +key, so the bucket's age is the deleter for this release. The end of a settle and the end of S15's fortnight wake the +healers through an in-process timer, armed by the act; at start, once the lease is first read, every settle still +running is re-armed from `attempts` and one that ended while they were down is judged at once, and the hand-act log's +watch hands over every entry again, which re-arms S15. So a restart loses no wake. + S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a fortnight; a drill and a person's decision by design never count. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index 170f0442..e18d8cae 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -362,8 +362,49 @@ func (h *Healers) tell(e Acted) { } } +// Rearm is the healers' start, once the lease is first read: every settle still running is re-armed from the attempts +// kept, and one whose end passed while the healers were down is judged at once, so a restart loses no wake. S15's +// fortnight end is re-armed by the hand-act log's watch, which hands over every current entry when it starts. +func (h *Healers) Rearm() { + all, err := h.attempts() + if err != nil { + h.Logf("[%s] what the healers did cannot be read, so no settle is re-armed: %v", Module, err) + return + } + now := h.Now() + newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first + for _, a := range all { + if a.Outcome != OutcomeEscalated { + newest[a.Healer+" "+a.BudgetKey] = a + } + } + var due []string + h.mu.Lock() + for _, a := range newest { + var row Healer + for _, r := range h.Registry { + if r.ID == a.Healer && r.repair != nil { + row = r + } + } + if row.ID == "" { + continue + } + if end := a.Begun.Add(row.Settle); end.After(now) { + h.wake(a.BudgetKey, end, a.Condition) + } else if now.Sub(a.Begun) <= row.Window { + due = append(due, a.Condition) + } + } + h.mu.Unlock() + sort.Strings(due) + for _, key := range due { + h.again(key) + } +} + // wake judges a condition again when its settle ends: the one wait of the healers, set by an act, never a look on a -// schedule. The settle is read from the attempt's time, so a wake lost to a restart is the next event's to make. +// schedule. The settle is read from the attempt's time, and Rearm sets it again after a restart. func (h *Healers) wake(budget string, at time.Time, key string) { if h.After == nil || !h.woken[budget].Before(at) { return diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index b3c4fb6a..00612cb8 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -98,6 +98,13 @@ type world struct { func newWorld() *world { w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.fresh() + return w +} + +// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. +func (w *world) fresh() { + w.wakes = nil w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, Now: func() time.Time { return w.now }, Ask: func(key string, body any) (json.RawMessage, error) { @@ -112,7 +119,6 @@ func newWorld() *world { return nil }, After: func(d time.Duration, _ func()) { w.wakes = append(w.wakes, d) }} - return w } func (w *world) handAway() { @@ -182,23 +188,40 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { w := newWorld() w.handAway() c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) steps := []struct { name string after time.Duration cond Condition attempts int outcome string // the newest said outcome, empty for none new + // restart starts the healers again over the same buckets instead of an event, and wake is the settle end + // they must re-arm (none when zero). + restart bool + wake time.Duration }{ - {"acts", 0, c, 1, OutcomeActed}, - {"within its settle: waits", 2 * time.Minute, c, 1, ""}, - {"after its settle, budget spent: escalates", 5 * time.Minute, c, 2, OutcomeEscalated}, + {"acts", 0, c, 1, OutcomeActed, false, 0}, + {"within its settle: waits", 2 * time.Minute, c, 1, "", false, 0}, + {"a restart within its settle re-arms its end", time.Minute, c, 1, "", true, 3*time.Minute + time.Second}, + {"a restart past its settle judges it at once: escalates", 4 * time.Minute, c, 2, OutcomeEscalated, true, 0}, {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", - Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, ""}, + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", false, 0}, } + var first []time.Duration for _, s := range steps { w.now = w.now.Add(s.after) before := len(w.said) - if _, err := w.h.Judge(s.cond, ""); err != nil { + if s.restart { + if len(first) == 0 { + first = w.wakes + } + w.fresh() + w.handAway() + w.h.Rearm() + if s.wake != 0 && !slices.Equal(w.wakes, []time.Duration{s.wake}) { + t.Errorf("%s: wakes %v, want %v", s.name, w.wakes, s.wake) + } + } else if _, err := w.h.Judge(s.cond, ""); err != nil { t.Fatalf("%s: %v", s.name, err) } if n := len(w.store.keys(AttemptsState)); n != s.attempts { @@ -211,8 +234,8 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) } } - if len(w.wakes) == 0 || w.wakes[0] != 6*time.Minute+time.Second { - t.Errorf("the settle's end wakes %v, want once after 6m1s", w.wakes) + if len(first) == 0 || first[0] != 6*time.Minute+time.Second { + t.Errorf("the settle's end wakes %v, want once after 6m1s", first) } // A second instance cannot begin the act an attempt's key already holds. x := newWorld() diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 86ec7539..2207e3c4 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" @@ -63,9 +64,12 @@ func main() { Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Emit: func(event string, body any) error { return stdio.Emit(event, body) }, After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }} - // The lease first: until it is read the healers write would-acts only. + // The lease first: until it is read the healers write would-acts only. Once it is first read, every settle is + // re-armed from the attempts kept, so a restart loses no wake. + var started sync.Once go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { h.LeaseHeard(c.Value, c.Op == "put", nil) + started.Do(func() { go h.Rearm() }) return nil }) go watched(HandActsRead, "", func(c stdio.StateChange) error {