Have anchor report on tier 1 inside its send in the rollout test, so its report never races the silence bound (issue 1938, met twice)

Under a loaded run the report came 5.7 s and then more than 10 s after the send, and anchor was named silent. Now the
fake send makes anchor's report as soon as tier 1 is recorded, and the test waits for it; laptop never reports and is
named silent.
This commit is contained in:
2026-10-24 07:13:22 +02:00
parent 9088db9a04
commit 5f2da0af69
+43 -8
View File
@@ -31,7 +31,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
// A quarter of an hour of silence is two seconds here. // A quarter of an hour of silence is two seconds here.
wasSilent := applySilentAfter wasSilent := applySilentAfter
applySilentAfter = 10 * time.Second // anchor reported 5.7 s after its send while every package ran at once (issue 1938) // Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's
// report never races it: anchor answers tier 1 inside its send (reportOnSend below).
applySilentAfter = 10 * time.Second
t.Cleanup(func() { applySilentAfter = wasSilent }) t.Cleanup(func() { applySilentAfter = wasSilent })
js, err := broker.Dial(testbus.URL(t)) js, err := broker.Dial(testbus.URL(t))
@@ -70,6 +72,8 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
var sends [][]string var sends [][]string
asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id
digests := 0 digests := 0
var reportOnSend func(node string) // set below, once a report can be made
anchorReported := make(chan error, 1)
wasSend := sendRollout wasSend := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
mu.Lock() mu.Lock()
@@ -89,6 +93,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
return nil, err return nil, err
} }
} }
// anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by
// order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports.
if len(sends) == 2 && reportOnSend != nil {
for _, node := range names {
if node == "anchor" {
go reportOnSend(node)
}
}
}
return names, nil return names, nil
} }
t.Cleanup(func() { sendRollout = wasSend }) t.Cleanup(func() { sendRollout = wasSend })
@@ -147,6 +160,26 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
// tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal.
tell := func(node string) error {
last, found, err := inv.SentTo(ctx, node)
if err != nil || !found {
return fmt.Errorf("%s was sent nothing: %v", node, err)
}
mu.Lock()
seq++
n := seq
mu.Unlock()
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
"report_sequence": n})
var report link.Report
if err := json.Unmarshal(body, &report); err != nil {
return err
}
_, err = listener.Heard(ctx, report)
return err
}
reportOnSend = func(node string) { anchorReported <- tell(node) }
heldKeys := func() map[silentWait]time.Time { heldKeys := func() map[silentWait]time.Time {
c := silentClockOf() c := silentClockOf()
if c == nil { if c == nil {
@@ -220,13 +253,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
defer mu.Unlock() defer mu.Unlock()
return len(sends) == 2 return len(sends) == 2
}) })
// anchor reports on tier 1's send only once the plan keeps when it was sent: a report heard before that reads as one // anchor reported on tier 1 inside its send.
// on the earlier send, and anchor would be named silent (a machine applies for longer than a save takes). select {
eventually(t, "tier 1's send kept in the plan", func() bool { case err := <-anchorReported:
b := plan().Modules["b"] if err != nil {
return b != nil && b.SentAt != nil t.Fatalf("anchor's report on tier 1: %v", err)
}) }
reports("anchor") case <-time.After(15 * time.Second):
t.Fatal("anchor never reported on tier 1")
}
// laptop never reports: its key expires and the new controller names it silent, which ends the rollout. // laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool { eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {