diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index ce92f043..5818cff8 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -31,7 +31,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { // A quarter of an hour of silence is two seconds here. wasSilent := applySilentAfter - applySilentAfter = 10 * time.Second // anchor reported 5.7 s after its send while every package ran at once (issue 1938) + // Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's + // report never races it: anchor answers tier 1 inside its send (reportOnSend below). + applySilentAfter = 10 * time.Second t.Cleanup(func() { applySilentAfter = wasSilent }) js, err := broker.Dial(testbus.URL(t)) @@ -70,6 +72,8 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { var sends [][]string asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id digests := 0 + var reportOnSend func(node string) // set below, once a report can be made + anchorReported := make(chan error, 1) wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { mu.Lock() @@ -89,6 +93,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { return nil, err } } + // anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by + // order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports. + if len(sends) == 2 && reportOnSend != nil { + for _, node := range names { + if node == "anchor" { + go reportOnSend(node) + } + } + } return names, nil } t.Cleanup(func() { sendRollout = wasSend }) @@ -147,6 +160,26 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { t.Fatal(err) } } + // tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal. + tell := func(node string) error { + last, found, err := inv.SentTo(ctx, node) + if err != nil || !found { + return fmt.Errorf("%s was sent nothing: %v", node, err) + } + mu.Lock() + seq++ + n := seq + mu.Unlock() + body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, + "report_sequence": n}) + var report link.Report + if err := json.Unmarshal(body, &report); err != nil { + return err + } + _, err = listener.Heard(ctx, report) + return err + } + reportOnSend = func(node string) { anchorReported <- tell(node) } heldKeys := func() map[silentWait]time.Time { c := silentClockOf() if c == nil { @@ -220,13 +253,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { defer mu.Unlock() return len(sends) == 2 }) - // anchor reports on tier 1's send only once the plan keeps when it was sent: a report heard before that reads as one - // on the earlier send, and anchor would be named silent (a machine applies for longer than a save takes). - eventually(t, "tier 1's send kept in the plan", func() bool { - b := plan().Modules["b"] - return b != nil && b.SentAt != nil - }) - reports("anchor") + // anchor reported on tier 1 inside its send. + select { + case err := <-anchorReported: + if err != nil { + t.Fatalf("anchor's report on tier 1: %v", err) + } + case <-time.After(15 * time.Second): + t.Fatal("anchor never reported on tier 1") + } // laptop never reports: its key expires and the new controller names it silent, which ends the rollout. eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {