mesh-catalog/modules/messenger/cmd/messenger/restart.go

package main

// Answers lost on a channel (novox/hq issue 384). On 2026-10-10 the desk's notifier started again and the
// desk showed, from its own memory, nine old popups: resolved, silenced and cancelled ones among them. A
// channel's memory is which notification a handle has, good for replacing or editing in place and for
// nothing else; what is still open and unanswered is known here alone. So a channel whose messages' answers
// were lost — the desk's notifier bundle started again: its notifications stay on screen, their buttons wait
// for nobody — says when in its standing (AnswersLostAt), and this router says again, on that kind only:
//
//   - **what is open and asked there**, and was last said before the answers were lost: one message per
//     problem, said again under its handle with the words and the answers of now, so the channel replaces its
//     message in place and its buttons work again. Sent to that kind directly — never through the outbox, whose
//     failure path moves a warning to the channel that reaches the operator away;
//   - **what is silenced**: its message edited to the silenced words, with no answers and no urgency;
//   - **nothing that ended**: a resolved or cancelled message's end was said where its message is; it never
//     comes back.
//
// Whether a record is said again is its own: last said before the answers were lost. So a router that started
// after the loss still says it, and a standing that repeats the time says nothing more (the record was said
// since). A kind that cannot send waits: what is open there is said once it can.
//
// **A restart says nothing it has said** (novox/hq ADR 0308 §6, issue 531). On 2026-10-12 every delivery of the
// tool runner made this pass twice, a second apart, each time editing the four silenced messages on the desk,
// which showed them again — one as "A mesh message was withheld". Three causes, each closed here:
//
//   - **Twice:** the desk's standing says the latest start of any machine's notifier, and the notifiers of two
//     machines started a second apart, so two standings with two times came, and each was acted on at once.
//     Now a loss is acted on once it stands for LostSettle: the standings of one restart are one pass.
//   - **Again after the router's own restart:** which loss was acted on was kept in memory, so a router that
//     started again acted on the standing it had already acted on. Now it is kept in the module's state
//     (`sent`, key `passes`), read back on start.
//   - **What it said:** a silenced message was edited to its silenced words, and a message withheld for its
//     content was said again as withheld. Now the pass says again only what is open, asked there and never
//     answered, whose words may leave; a silenced, a withheld or an answered message is left as it is. Being
//     held by a walk is no reason: a message with an open ask is never held (ADR 0308 §3), even one a walk took
//     before its ask came, so its answers are made to work again like any other's.

import (
	"fmt"
	"sort"
	"time"
)

// LostSettle: a loss of answers is acted on once no later loss came for this long — the standings of one
// restart, a notifier on each machine, are one pass.
const LostSettle = 10 * time.Second

// Pass is the restart pass made in full on a kind: the loss it answered and when it was made.
type Pass struct {
	Lost time.Time `json:"lost"`
	At   time.Time `json:"at"`
}

// Stand takes what a kind said of itself, and notes answers lost there since this router last acted.
func (h *Holder) Stand(kind string, s Standing) {
	if h.Bench != nil {
		h.Bench.Stand(kind, s)
	}
	if s.AnswersLostAt.IsZero() {
		return
	}
	h.mu.Lock()
	defer h.mu.Unlock()
	h.init()
	if !s.AnswersLostAt.After(h.passes[kind].Lost) || !s.AnswersLostAt.After(h.lost[kind]) {
		return // acted on, or noted already
	}
	h.lost[kind] = s.AnswersLostAt
}

// duePasses makes the pass of each kind whose loss of answers stood for LostSettle, once per loss.
func (h *Holder) duePasses(now time.Time) {
	h.mu.Lock()
	due := map[string]time.Time{}
	for kind, lost := range h.lost {
		if now.Sub(lost) >= LostSettle {
			due[kind] = lost
		}
	}
	h.mu.Unlock()
	for kind, lost := range due {
		ch := h.channelOf(kind)
		if ch == nil {
			continue // the controller's word on the kinds is not read yet: acted on later
		}
		if err := ch.Ready(); err != nil {
			h.mu.Lock()
			said := h.saidOnce["lost."+kind]
			h.mu.Unlock()
			if said.IsZero() || now.Sub(said) >= 10*time.Minute {
				h.Logf("[messenger] %s lost its answers at %s and cannot send (%v): what is open there is said again once it can",
					kind, lost.UTC().Format(time.RFC3339), err)
				h.mu.Lock()
				h.saidOnce["lost."+kind] = now
				h.mu.Unlock()
			}
			continue
		}
		h.mu.Lock()
		tried := h.saidOnce["lost-try."+kind]
		h.mu.Unlock()
		if !tried.IsZero() && now.Sub(tried) < RetryEvery {
			continue
		}
		if !h.reoffer(ch, lost) {
			h.mu.Lock()
			h.saidOnce["lost-try."+kind] = now // what failed is tried again in a minute
			h.mu.Unlock()
			continue
		}
		h.mu.Lock()
		delete(h.saidOnce, "lost-try."+kind)
		h.mu.Unlock()
		h.mu.Lock()
		h.passes[kind] = Pass{Lost: lost, At: now}
		if h.lost[kind].Equal(lost) {
			delete(h.lost, kind)
		}
		passes := map[string]Pass{}
		for k, p := range h.passes {
			passes[k] = p
		}
		h.mu.Unlock()
		if k, ok := h.Store.(walkKeeper); ok {
			if err := k.PutPasses(passes); err != nil {
				h.noteStore(err)
			}
		}
	}
}

// channelOf is the channel sending to a kind now, or nil.
func (h *Holder) channelOf(kind string) Channel {
	for _, ch := range h.channels() {
		if ch.Name() == kind {
			return ch
		}
	}
	return nil
}

// reoffer says again on one channel, whose answers were lost at `lost`, what is still open, asked there and
// never answered, and leaves everything else as it is: what ended, what is silenced, what was withheld for its
// content, and what was answered (ADR 0308 §6). It answers whether every message to be said again was.
func (h *Holder) reoffer(ch Channel, lost time.Time) bool {
	kind := ch.Name()
	rp, canReplace := ch.(replacer)
	now := h.Now()
	h.mu.Lock()
	var recs []*Record
	for _, r := range h.open {
		recs = append(recs, r)
	}
	h.mu.Unlock()
	sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) })
	again, left, ended, failed := 0, 0, 0, 0
	for _, r := range recs {
		h.mu.Lock()
		id, shown := r.Sent[kind]
		open := r.Cleared.IsZero()
		before := lastSaid(r).Before(lost)
		reminded := r.Reminded
		withheld, answered := r.Refused != "", r.Answered != ""
		h.mu.Unlock()
		if !shown || id == "" {
			continue // never a message of its own there: nothing to say again under
		}
		switch {
		case !open:
			ended++ // its end is said where its message is; nothing comes back
		case r.silenced(now), withheld, answered:
			left++ // a silence, a withheld message or an answer is never said again for a restart
		case h.askKinds(r)[kind] && before && canReplace:
			what := "raised"
			if reminded {
				what = "reminder"
			}
			err := rp.Replace(id, h.sayOn(r, what, kind))
			h.noteChannel(kind, err)
			if err != nil {
				h.Logf("[messenger] could not say %s again on %s (%v): tried again at its next standing", r.Key, kind, err)
				failed++
				continue
			}
			h.mu.Lock()
			r.LastSaid = now
			h.lastSend[kind] = now
			h.mu.Unlock()
			h.record(Sent{At: now, Channel: kind, Key: r.Key, What: what + ", again: its answers were lost", Outcome: "sent"})
			h.persist(r)
			again++
		}
	}
	h.record(Sent{At: now, Channel: kind, Key: "(answers lost)", What: "at " + lost.UTC().Format(time.RFC3339),
		Outcome: fmt.Sprintf("%d open and asked said again, %d silenced, withheld or answered left as they are, "+
			"%d ended left ended, %d failed", again, left, ended, failed)})
	h.Logf("[messenger] %s lost its answers at %s: %d open and asked said again in place, %d silenced, withheld or "+
		"answered left as they are, %d ended left ended, %d failed; nothing shown from the channel's memory", kind,
		lost.UTC().Format(time.RFC3339), again, left, ended, failed)
	return failed == 0
}