Merge remote-tracking branch 'origin/fix/1931-reconnects-verb' into fix/1937-done-key-blobs

cmd/mesh-controller/rollout_events_test.go

@@ -31,7 +31,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 
 	// A quarter of an hour of silence is two seconds here.
 	wasSilent := applySilentAfter
-	applySilentAfter = 10 * time.Second // anchor reported 5.7 s after its send while every package ran at once (issue 1938)
+	// Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's
+	// report never races it: anchor answers tier 1 inside its send (reportOnSend below).
+	applySilentAfter = 10 * time.Second
 	t.Cleanup(func() { applySilentAfter = wasSilent })
 
 	js, err := broker.Dial(testbus.URL(t))
@@ -70,6 +72,8 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 	var sends [][]string
 	asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id
 	digests := 0
+	var reportOnSend func(node string) // set below, once a report can be made
+	anchorReported := make(chan error, 1)
 	wasSend := sendRollout
 	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
 		mu.Lock()
@@ -89,6 +93,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 				return nil, err
 			}
 		}
+		// anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by
+		// order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports.
+		if len(sends) == 2 && reportOnSend != nil {
+			for _, node := range names {
+				if node == "anchor" {
+					go reportOnSend(node)
+				}
+			}
+		}
 		return names, nil
 	}
 	t.Cleanup(func() { sendRollout = wasSend })
@@ -147,6 +160,26 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
+	// tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal.
+	tell := func(node string) error {
+		last, found, err := inv.SentTo(ctx, node)
+		if err != nil || !found {
+			return fmt.Errorf("%s was sent nothing: %v", node, err)
+		}
+		mu.Lock()
+		seq++
+		n := seq
+		mu.Unlock()
+		body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
+			"report_sequence": n})
+		var report link.Report
+		if err := json.Unmarshal(body, &report); err != nil {
+			return err
+		}
+		_, err = listener.Heard(ctx, report)
+		return err
+	}
+	reportOnSend = func(node string) { anchorReported <- tell(node) }
 	heldKeys := func() map[silentWait]time.Time {
 		c := silentClockOf()
 		if c == nil {
@@ -220,13 +253,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
 		defer mu.Unlock()
 		return len(sends) == 2
 	})
-	// anchor reports on tier 1's send only once the plan keeps when it was sent: a report heard before that reads as one
-	// on the earlier send, and anchor would be named silent (a machine applies for longer than a save takes).
-	eventually(t, "tier 1's send kept in the plan", func() bool {
-		b := plan().Modules["b"]
-		return b != nil && b.SentAt != nil
-	})
-	reports("anchor")
+	// anchor reported on tier 1 inside its send.
+	select {
+	case err := <-anchorReported:
+		if err != nil {
+			t.Fatalf("anchor's report on tier 1: %v", err)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("anchor never reported on tier 1")
+	}
 
 	// laptop never reports: its key expires and the new controller names it silent, which ends the rollout.
 	eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool {