Retire the controller's merges consumer and read merges from the forge bridge's stream, so a held merge can never again queue every merge or look like a deaf controller (issue 774)
The controller-merges consumer (no max-deliver, one at a time) held the de-spiegel merge e25e1beb from 10-29 09:39: keepForgeMerge put an announcement with its Paths into delivery-merges (16 KiB values), got 10054, retried "not now" forever, and 167 merges queued behind it. Every five-minute hold outran the deaf bound, so core.controller.deaf flapped. Merges are mesh-delivery's: the gate merges on the Approve and the forge bridge says each merge on its stream FORGE (pull request, head, merge commit), kept a week. The controller's copy was a leftover, read only at a moved module's registration (ADR 0420 §7, ADR 0432). That read now finds the merge on FORGE; the consumer, its follow, its handler and its hold delays are removed, and every assertion removes the retired consumer from the bus. Kind fix for the watchdog: the event loop records when a message it held back is due again, and deafness counts from then, so no hold longer than the bound (on any consumer) reads as deafness. Quick fix: strip Paths from the kept announcement and add max-deliver with a dead letter. Thorough fix: remove the duplicate duty and make the deaf watchdog hold-aware. Chose thorough: the controller is being deprecated and the merges already have an owner; the hold-vs-deaf confusion is a kind, not one consumer. Research 072 piece: 03-domains (Control shrinks to placement and composition; the delivery parts own what a merge is) and milestone 3 (health signals stop sharing the controller's loop assumptions).
This commit is contained in:
@@ -62,7 +62,7 @@ func Nats(js *broker.JetStream) Inbound {
|
||||
// whatever was asked for — and not at all when nothing was.
|
||||
func (n *natsInbound) Also(kind string) error {
|
||||
switch kind {
|
||||
case KindModuleMoved, KindCatchUp, KindSourceMoved, KindProvisioner, KindPullUpdated, KindDecided,
|
||||
case KindModuleMoved, KindCatchUp, KindProvisioner, KindPullUpdated, KindDecided,
|
||||
KindDeliveryMerged:
|
||||
n.follows[kind] = true
|
||||
return nil
|
||||
@@ -145,24 +145,6 @@ func (n *natsInbound) Receive(ctx context.Context, act func(context.Context, Con
|
||||
defer func() { _ = followed.Unsubscribe() }()
|
||||
}
|
||||
|
||||
// And the forge's merges, off a consumer of their own: no max-deliver and no reset, so a merge held for the
|
||||
// plans waits for them however long (novox/hq ADR 0407 decision 3). Bound to the same channel: one loop.
|
||||
if n.follows[KindSourceMoved] {
|
||||
if events == nil {
|
||||
events = make(chan *nats.Msg, Prefetch)
|
||||
}
|
||||
merges, err := standingBy(ctx, log.Default(), "EVENTS", func() (*nats.Subscription, error) {
|
||||
return js.ChanSubscribe(MergedSubject, events, nats.Bind("EVENTS", broker.MergesConsumerName))
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribing to the forge's merges: %w", err)
|
||||
}
|
||||
if merges == nil {
|
||||
return nil
|
||||
}
|
||||
defer func() { _ = merges.Unsubscribe() }()
|
||||
}
|
||||
|
||||
// A connection that dropped is said, not discovered. A controller whose bus connection is gone
|
||||
// is a mesh where nothing can be told anything.
|
||||
gone := make(chan error, 1)
|
||||
@@ -423,7 +405,11 @@ func (m *natsControl) Hold(after time.Duration) error {
|
||||
if _, already := m.on.since[m.seq]; !already {
|
||||
m.on.since[m.seq] = time.Now()
|
||||
}
|
||||
return m.msg.NakWithDelay(after)
|
||||
if err := m.msg.NakWithDelay(after); err != nil {
|
||||
return err
|
||||
}
|
||||
Loop.HeldUntil(time.Now().Add(after))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *natsControl) forget() {
|
||||
|
||||
Reference in New Issue
Block a user