jschoubben
48d9f22fa5
The controller-merges consumer (no max-deliver, one at a time) held the de-spiegel merge e25e1beb from 10-29 09:39: keepForgeMerge put an announcement with its Paths into delivery-merges (16 KiB values), got 10054, retried "not now" forever, and 167 merges queued behind it. Every five-minute hold outran the deaf bound, so core.controller.deaf flapped. Merges are mesh-delivery's: the gate merges on the Approve and the forge bridge says each merge on its stream FORGE (pull request, head, merge commit), kept a week. The controller's copy was a leftover, read only at a moved module's registration (ADR 0420 §7, ADR 0432). That read now finds the merge on FORGE; the consumer, its follow, its handler and its hold delays are removed, and every assertion removes the retired consumer from the bus. Kind fix for the watchdog: the event loop records when a message it held back is due again, and deafness counts from then, so no hold longer than the bound (on any consumer) reads as deafness. Quick fix: strip Paths from the kept announcement and add max-deliver with a dead letter. Thorough fix: remove the duplicate duty and make the deaf watchdog hold-aware. Chose thorough: the controller is being deprecated and the merges already have an owner; the hold-vs-deaf confusion is a kind, not one consumer. Research 072 piece: 03-domains (Control shrinks to placement and composition; the delivery parts own what a merge is) and milestone 3 (health signals stop sharing the controller's loop assumptions).
71 lines
2.5 KiB
Go
71 lines
2.5 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
)
|
|
|
|
// What a store restarting under an adoption answers with (novox/hq issues 082, 083).
|
|
var restarting = &pgconn.PgError{Code: "57P03", Message: "the database system is starting up"}
|
|
|
|
type recordsWith struct{ err error }
|
|
|
|
func (r recordsWith) Built(context.Context, BuildResult) error { return r.err }
|
|
|
|
type upgradesWith struct{ err error }
|
|
|
|
func (u upgradesWith) Upgraded(context.Context, Upgraded) error { return u.err }
|
|
|
|
type replaysWith struct{ err error }
|
|
|
|
func (r replaysWith) Announceable(context.Context) ([]Announcement, error) { return nil, r.err }
|
|
|
|
// An upgrade announcement arriving while the store restarts is asked again; any other failure is
|
|
// acknowledged, so it cannot stop every upgrade behind it.
|
|
func TestAnUpgradeWaitsOutARestartingStoreAndNothingElse(t *testing.T) {
|
|
moved := Upgraded{Module: "gitea", Commit: "abcdef0123"}
|
|
for _, c := range []struct {
|
|
what string
|
|
err error
|
|
want func(*settled) bool
|
|
}{
|
|
{"the store away, said by the upgrader", errors.Join(ErrTryAgain, restarting), func(s *settled) bool { return s.unsettled() }},
|
|
{"a push that timed out", context.DeadlineExceeded, func(s *settled) bool { return s.acked && !s.nacked }},
|
|
{"cannot act", errors.New("anchor cannot be resolved"), func(s *settled) bool { return s.acked && !s.nacked }},
|
|
{"acted", nil, func(s *settled) bool { return s.acked && !s.nacked }},
|
|
} {
|
|
s, in := serving()
|
|
s.upgrader = upgradesWith{err: c.err}
|
|
to := &settled{}
|
|
s.act(context.Background(), in.sends(t, to, KindModuleMoved, moved))
|
|
if !c.want(to) {
|
|
t.Errorf("%s: an upgrade was settled as %+v", c.what, *to)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A catalogue's request to catch up is acknowledged after the work, and asked again while the
|
|
// store cannot be read — not lost until the catalogue next restarts.
|
|
func TestACatchUpWaitsOutARestartingStore(t *testing.T) {
|
|
for _, c := range []struct {
|
|
what string
|
|
err error
|
|
want func(*settled) bool
|
|
}{
|
|
{"restarting", restarting, func(s *settled) bool { return s.unsettled() }},
|
|
{"unreadable", errors.New("a build row is malformed"), func(s *settled) bool { return s.acked && !s.nacked }},
|
|
{"nothing to replay", nil, func(s *settled) bool { return s.acked && !s.nacked }},
|
|
} {
|
|
s, in := serving()
|
|
s.replayer = replaysWith{err: c.err}
|
|
to := &settled{}
|
|
s.act(context.Background(), in.sends(t, to, KindCatchUp, map[string]string{}))
|
|
if !c.want(to) {
|
|
t.Errorf("%s: a catch-up request was settled as %+v", c.what, *to)
|
|
}
|
|
}
|
|
}
|