Wait 20 s, not 5 s, for each step of the gate's path over the bus: a loaded build node failed the ask wait twice (issue 1404)

modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go

@@ -520,7 +520,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 	passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1"))
 
 	// No review names the head, so nothing is asked (ADR 0496, issue 1803): the delivery waits for a review.
-	rt.until("the delivery to wait for a review", 5*time.Second, func() bool {
+	rt.until("the delivery to wait for a review", eventWithin, func() bool {
 		e, err := b.kvs["pulls"].Get(ctx, "novox.app.7")
 		var k PullKey
 		return err == nil && json.Unmarshal(e.Value(), &k) == nil && k.Verdict == "pass"
@@ -537,7 +537,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 		"title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head})
 	_ = passed
 	var key string
-	rt.until("the ask to be written", 5*time.Second, func() bool { key = onlyAsk(b); return key != "" })
+	rt.until("the ask to be written", eventWithin, func() bool { key = onlyAsk(b); return key != "" })
 	ask := askAt(t, b, key)
 	t.Logf("the ask was written %s after the edit's event", time.Since(edit))
 	if !strings.Contains(ask.Ask.Explanation, "reviewed by lead on shanks") || ask.Ask.ID != sdk.AskID(key) {
@@ -547,7 +547,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 		t.Fatalf("merged before the Approve")
 	}
 	// The pull request's key holds its head, the trunk commit it was asked against and the verdict.
-	rt.until("the pull request's key to hold head, trunk and verdict", 5*time.Second, func() bool {
+	rt.until("the pull request's key to hold head, trunk and verdict", eventWithin, func() bool {
 		e, err := b.kvs["pulls"].Get(ctx, "novox.app.7")
 		var k PullKey
 		return err == nil && json.Unmarshal(e.Value(), &k) == nil && k == PullKey{Head: head, Trunk: "t1t1t1t1t1t1", Verdict: "pass"}
@@ -559,7 +559,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 			t.Fatalf("the gate asked %s %d times: the bridge writes to the forge", tool, n)
 		}
 	}
-	rt.until("the owed keys to be written", 5*time.Second, func() bool {
+	rt.until("the owed keys to be written", eventWithin, func() bool {
 		keys, err := b.kvs["owed"].Keys(ctx)
 		return err == nil && len(keys) > 0
 	})
@@ -567,7 +567,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 	// seconds, with the Approve's heads, and the gate asks the forge nothing.
 	approved := b.answer(t, key, OptionApprove)
 	var handed sdk.Merge
-	rt.until("the merge to be handed to the bridge", 5*time.Second, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 })
+	rt.until("the merge to be handed to the bridge", eventWithin, func() bool { handed = mergeAt(b, "fix/x"); return len(handed.Pulls) > 0 })
 	if len(handed.Pulls) != 1 || handed.Pulls[0].Head.Head != head || handed.Pulls[0].Number != 7 || handed.Pulls[0].State != sdk.MergeMerging ||
 		handed.Approved[sdk.ApprovedKey("novox/app", 7)] != head || handed.Ask != key {
 		t.Fatalf("the merge handed is %+v; wanted pull request 7 at the approved head, merging, under the ask %s", handed, key)
@@ -579,7 +579,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
 	// The bridge says it merged; the mesh's source then holds the merge, in one write.
 	b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40),
 		Root: "r", Trees: map[string]string{"app": "t"}})
-	rt.until("source.main to hold the merge", 5*time.Second, func() bool {
+	rt.until("source.main to hold the merge", eventWithin, func() bool {
 		e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey)
 		if err != nil {
 			return false
@@ -743,7 +743,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
 	// One member is handed, with both heads approved, within seconds of the answer; the second only after the bridge says the
 	// first merged.
 	var first sdk.Merge
-	rt.until("the first member to be handed", 5*time.Second, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 })
+	rt.until("the first member to be handed", eventWithin, func() bool { first = mergeAt(b, "feat/x"); return len(first.Pulls) > 0 })
 	if len(first.Pulls) != 1 || first.Pulls[0].State != sdk.MergeMerging || len(first.Approved) != 2 ||
 		first.Approved[sdk.ApprovedKey("novox/mesh-controller", 1)] != ctlHead || first.Approved[sdk.ApprovedKey("novox/mesh-catalog", 2)] != catHead {
 		t.Fatalf("the first merge handed is %+v", first)
@@ -757,14 +757,14 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe
 	b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: one.Repository, Number: one.Number, Head: one.Head.Head,
 		Merge: strings.Repeat("a", 40), Root: "r", Trees: map[string]string{"x": "t"}})
 	var both sdk.Merge
-	rt.until("the second member to be handed", 5*time.Second, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 })
+	rt.until("the second member to be handed", eventWithin, func() bool { both = mergeAt(b, "feat/x"); return len(both.Pulls) == 2 })
 	if both.Pulls[0].State != sdk.MergeMerged || both.Pulls[1].State != sdk.MergeMerging || both.Pulls[1].Repository == one.Repository {
 		t.Fatalf("the second merge handed is %+v", both.Pulls)
 	}
 	two := both.Pulls[1]
 	b.forgeSays(t, sdk.Forge{Kind: sdk.ForgeMerged, Repository: two.Repository, Number: two.Number, Head: two.Head.Head,
 		Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"x": "t"}})
-	rt.until("source.main to hold both merges", 5*time.Second, func() bool {
+	rt.until("source.main to hold both merges", eventWithin, func() bool {
 		e, err := b.kvs["source"].Get(ctx, sdk.SourceMainKey)
 		if err != nil {
 			return false
@@ -942,3 +942,10 @@ func mustDecode[T any](t *testing.T, raw []byte) T {
 	}
 	return v
 }
+
+// eventWithin is how long a test waits for one step of the gate's path over the real bus. On an idle machine each step
+// takes milliseconds (the tests log it), but the build node runs the whole group's checks at once, and a step that
+// fails its first try (a key rewritten under it, a request that timed out) is tried again by the gate only after
+// retryFirst (10 s, effects.go), which the test cannot shorten without changing production code. 20 s covers that one
+// retry plus the load; "within seconds" stays true and a stuck path still fails (issue 1404).
+const eventWithin = 20 * time.Second