Re-arm every running settle from the attempts kept when the healers start, so a restart loses no wake (issue 1952)

A settle that ended while they were down is judged at once. The README says the week-long bucket age is the deleter of would and attempts for this release.
2026-10-29 21:49:46 +01:00
parent 885b7a3a8e
commit 9dd27d2edb
4 changed files with 84 additions and 10 deletions
@@ -18,6 +18,12 @@ mid-act has spent it and a second instance cannot begin it again. The condition
 **Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would
 take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition.

+Both buckets age as a whole, each key a week after its last write: the runtime gives a bundle no time to live per
+key, so the bucket's age is the deleter for this release. The end of a settle and the end of S15's fortnight wake the
+healers through an in-process timer, armed by the act; at start, once the lease is first read, every settle still
+running is re-armed from `attempts` and one that ended while they were down is judged at once, and the hand-act log's
+watch hands over every entry again, which re-arms S15. So a restart loses no wake.
+
 S15: it watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a
 fortnight; a drill and a person's decision by design never count.

@@ -362,8 +362,49 @@ func (h *Healers) tell(e Acted) {
 	}
 }

+// Rearm is the healers' start, once the lease is first read: every settle still running is re-armed from the attempts
+// kept, and one whose end passed while the healers were down is judged at once, so a restart loses no wake. S15's
+// fortnight end is re-armed by the hand-act log's watch, which hands over every current entry when it starts.
+func (h *Healers) Rearm() {
+	all, err := h.attempts()
+	if err != nil {
+		h.Logf("[%s] what the healers did cannot be read, so no settle is re-armed: %v", Module, err)
+		return
+	}
+	now := h.Now()
+	newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first
+	for _, a := range all {
+		if a.Outcome != OutcomeEscalated {
+			newest[a.Healer+" "+a.BudgetKey] = a
+		}
+	}
+	var due []string
+	h.mu.Lock()
+	for _, a := range newest {
+		var row Healer
+		for _, r := range h.Registry {
+			if r.ID == a.Healer && r.repair != nil {
+				row = r
+			}
+		}
+		if row.ID == "" {
+			continue
+		}
+		if end := a.Begun.Add(row.Settle); end.After(now) {
+			h.wake(a.BudgetKey, end, a.Condition)
+		} else if now.Sub(a.Begun) <= row.Window {
+			due = append(due, a.Condition)
+		}
+	}
+	h.mu.Unlock()
+	sort.Strings(due)
+	for _, key := range due {
+		h.again(key)
+	}
+}
+
 // wake judges a condition again when its settle ends: the one wait of the healers, set by an act, never a look on a
-// schedule. The settle is read from the attempt's time, so a wake lost to a restart is the next event's to make.
+// schedule. The settle is read from the attempt's time, and Rearm sets it again after a restart.
 func (h *Healers) wake(budget string, at time.Time, key string) {
 	if h.After == nil || !h.woken[budget].Before(at) {
 		return
@@ -98,6 +98,13 @@ type world struct {

 func newWorld() *world {
 	w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)}
+	w.fresh()
+	return w
+}
+
+// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten.
+func (w *world) fresh() {
+	w.wakes = nil
 	w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {},
 		Now: func() time.Time { return w.now },
 		Ask: func(key string, body any) (json.RawMessage, error) {
@@ -112,7 +119,6 @@ func newWorld() *world {
 			return nil
 		},
 		After: func(d time.Duration, _ func()) { w.wakes = append(w.wakes, d) }}
-	return w
 }

 func (w *world) handAway() {
@@ -182,23 +188,40 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
 	w := newWorld()
 	w.handAway()
 	c := lost("records")
+	_ = w.store.Put(ConditionsRead, c.Key, c)
 	steps := []struct {
 		name     string
 		after    time.Duration
 		cond     Condition
 		attempts int
 		outcome  string // the newest said outcome, empty for none new
+		// restart starts the healers again over the same buckets instead of an event, and wake is the settle end
+		// they must re-arm (none when zero).
+		restart bool
+		wake    time.Duration
 	}{
-		{"acts", 0, c, 1, OutcomeActed},
-		{"within its settle: waits", 2 * time.Minute, c, 1, ""},
-		{"after its settle, budget spent: escalates", 5 * time.Minute, c, 2, OutcomeEscalated},
+		{"acts", 0, c, 1, OutcomeActed, false, 0},
+		{"within its settle: waits", 2 * time.Minute, c, 1, "", false, 0},
+		{"a restart within its settle re-arms its end", time.Minute, c, 1, "", true, 3*time.Minute + time.Second},
+		{"a restart past its settle judges it at once: escalates", 4 * time.Minute, c, 2, OutcomeEscalated, true, 0},
 		{"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator",
-			Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, ""},
+			Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", false, 0},
 	}
+	var first []time.Duration
 	for _, s := range steps {
 		w.now = w.now.Add(s.after)
 		before := len(w.said)
-		if _, err := w.h.Judge(s.cond, ""); err != nil {
+		if s.restart {
+			if len(first) == 0 {
+				first = w.wakes
+			}
+			w.fresh()
+			w.handAway()
+			w.h.Rearm()
+			if s.wake != 0 && !slices.Equal(w.wakes, []time.Duration{s.wake}) {
+				t.Errorf("%s: wakes %v, want %v", s.name, w.wakes, s.wake)
+			}
+		} else if _, err := w.h.Judge(s.cond, ""); err != nil {
 			t.Fatalf("%s: %v", s.name, err)
 		}
 		if n := len(w.store.keys(AttemptsState)); n != s.attempts {
@@ -211,8 +234,8 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) {
 			t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome)
 		}
 	}
-	if len(w.wakes) == 0 || w.wakes[0] != 6*time.Minute+time.Second {
-		t.Errorf("the settle's end wakes %v, want once after 6m1s", w.wakes)
+	if len(first) == 0 || first[0] != 6*time.Minute+time.Second {
+		t.Errorf("the settle's end wakes %v, want once after 6m1s", first)
 	}
 	// A second instance cannot begin the act an attempt's key already holds.
 	x := newWorld()
@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sync"
 	"time"

 	stdio "git.novox.be/novox/mesh-sdk/go"
@@ -63,9 +64,12 @@ func main() {
 		Ask:   func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) },
 		Emit:  func(event string, body any) error { return stdio.Emit(event, body) },
 		After: func(d time.Duration, f func()) { time.AfterFunc(d, f) }}
-	// The lease first: until it is read the healers write would-acts only.
+	// The lease first: until it is read the healers write would-acts only. Once it is first read, every settle is
+	// re-armed from the attempts kept, so a restart loses no wake.
+	var started sync.Once
 	go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error {
 		h.LeaseHeard(c.Value, c.Op == "put", nil)
+		started.Do(func() { go h.Rearm() })
 		return nil
 	})
 	go watched(HandActsRead, "", func(c stdio.StateChange) error {