Review of 083: a message the store cannot take is held and retried on a ticker, not slept on, so enrolments are answered meanwhile; a newer one per subject supersedes; the password is replaced after the spend; the same presenter may finish after a lost answer; upgrades retry only on the store

internal/link/report_retry_test.go

@@ -20,85 +20,85 @@ func (a *saidTo) Nack(_ uint64, _ bool, requeue bool) error {
 	return nil
 }
 func (a *saidTo) Reject(uint64, bool) error { return nil }
+func (a *saidTo) unsettled() bool           { return !a.acked && !a.nacked }
 
 type heardWith struct{ err error }
 
 func (h heardWith) Heard(context.Context, Report) error { return h.err }
 
-func aReport(t *testing.T, to *saidTo) amqp.Delivery {
+// switchable answers with whatever it is set to — the store away, then back.
+type switchable struct{ err error }
+
+func (h *switchable) Heard(context.Context, Report) error { return h.err }
+
+var tag uint64
+
+func aReport(t *testing.T, to *saidTo, node, declared string) amqp.Delivery {
 	t.Helper()
-	body, err := json.Marshal(Report{Node: "anchor", Applied: []string{"store"}})
+	body, err := json.Marshal(Report{Node: node, Declared: declared, Applied: []string{"store"}})
 	if err != nil {
 		t.Fatal(err)
 	}
-	return amqp.Delivery{Acknowledger: to, RoutingKey: KeyReport, Body: body}
+	tag++
+	return amqp.Delivery{Acknowledger: to, RoutingKey: KeyReport, Body: body, DeliveryTag: tag}
 }
 
-// A report the store could not take right now goes back to the broker to be asked again; one the
-// store answered no to is acknowledged, or it would come back for ever (novox/hq issue 082).
-func TestAReportTheStoreCouldNotTakeIsKeptAndOneItRefusedIsNot(t *testing.T) {
-	quiet := log.New(io.Discard, "", 0)
+func quiet() *log.Logger { return log.New(io.Discard, "", 0) }
 
-	notNow := &saidTo{}
-	s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}, log: quiet, again: 1}
-	s.handleReport(context.Background(), aReport(t, notNow))
-	if notNow.acked || !notNow.nacked || !notNow.requeued {
-		t.Fatalf("a report the store could not take yet was not handed back to be asked again: %+v", notNow)
+// A report the store could not take right now is held, unsettled, and recorded when the store is
+// back; one the store answered no to is acknowledged; one recorded is acknowledged (issue 082, 083).
+func TestAReportTheStoreCouldNotTakeIsHeldAndOneItRefusedIsNot(t *testing.T) {
+	store := &switchable{err: errors.Join(ErrTryAgain, errors.New("starting up"))}
+	s := &Server{listener: store, log: quiet()}
+	held := &saidTo{}
+	s.handleReport(context.Background(), aReport(t, held, "anchor", "d1"))
+	if !held.unsettled() || len(s.parked) != 1 {
+		t.Fatalf("a report the store could not take was not held: %+v, %d held", held, len(s.parked))
+	}
+	store.err = nil
+	s.retryHeld(context.Background())
+	if !held.acked || len(s.parked) != 0 {
+		t.Fatalf("a held report was not recorded once the store was back: %+v, %d held", held, len(s.parked))
 	}
 
 	refused := &saidTo{}
-	s = &Server{listener: heardWith{err: errors.New("a report named no node")}, log: quiet, again: 1}
-	s.handleReport(context.Background(), aReport(t, refused))
+	s = &Server{listener: heardWith{err: errors.New("a report named no node")}, log: quiet()}
+	s.handleReport(context.Background(), aReport(t, refused, "anchor", "d1"))
 	if !refused.acked || refused.nacked {
-		t.Fatalf("a report the store answered no to was not acknowledged, so it would spin: %+v", refused)
+		t.Fatalf("a report the store answered no to was not acknowledged: %+v", refused)
 	}
+}
 
-	recorded := &saidTo{}
-	s = &Server{listener: heardWith{}, log: quiet}
-	s.handleReport(context.Background(), aReport(t, recorded))
-	if !recorded.acked || recorded.nacked {
-		t.Fatalf("a recorded report was not acknowledged: %+v", recorded)
+// A newer report from the same node supersedes one of its reports still held: recorded after the
+// newer, the older would overwrite what the node is doing now.
+func TestANewerReportSupersedesAHeldOneFromTheSameNode(t *testing.T) {
+	s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))}, log: quiet()}
+	older, newer, other := &saidTo{}, &saidTo{}, &saidTo{}
+	s.handleReport(context.Background(), aReport(t, older, "anchor", "d1"))
+	s.handleReport(context.Background(), aReport(t, other, "laptop", "d7"))
+	s.handleReport(context.Background(), aReport(t, newer, "anchor", "d2"))
+	if !older.acked {
+		t.Fatalf("the older report was not set aside by the newer: %+v", older)
+	}
+	if !newer.unsettled() || !other.unsettled() || len(s.parked) != 2 {
+		t.Fatalf("the newer report and another node's were not both held: newer %+v other %+v, %d held",
+			newer, other, len(s.parked))
 	}
 }
 
 // A store that has not come back within the bound is not restarting: the report is let go, loudly,
-// rather than holding every enrolment and report behind it for ever (novox/hq issue 082, review).
+// rather than held for ever.
 func TestAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
-	quiet := log.New(io.Discard, "", 0)
 	s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("connection refused"))},
-		log: quiet, again: 1, giveUp: time.Millisecond}
-
-	first := &saidTo{}
-	s.handleReport(context.Background(), aReport(t, first))
-	if !first.requeued {
-		t.Fatalf("the first failure was not handed back: %+v", first)
+		log: quiet(), giveUp: time.Millisecond}
+	held := &saidTo{}
+	s.handleReport(context.Background(), aReport(t, held, "anchor", "d1"))
+	if !held.unsettled() {
+		t.Fatalf("the first failure was not held: %+v", held)
 	}
 	time.Sleep(5 * time.Millisecond)
-	later := &saidTo{}
-	s.handleReport(context.Background(), aReport(t, later))
-	if !later.acked || later.nacked {
-		t.Fatalf("a report past the bound was not let go, so it would hold the queue for ever: %+v", later)
-	}
-	// Let go, and forgotten: the same report arriving fresh starts a new clock.
-	again := &saidTo{}
-	s.handleReport(context.Background(), aReport(t, again))
-	if !again.requeued {
-		t.Fatalf("a report let go was not forgotten, so its next arrival is given no chance: %+v", again)
-	}
-}
-
-// Shutting down does not wait out the pause.
-func TestAReportBeingTriedAgainDoesNotHoldUpShutdown(t *testing.T) {
-	quiet := log.New(io.Discard, "", 0)
-	s := &Server{listener: heardWith{err: errors.Join(ErrTryAgain, errors.New("starting up"))},
-		log: quiet, again: time.Hour}
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	done := make(chan struct{})
-	go func() { s.handleReport(ctx, aReport(t, &saidTo{})); close(done) }()
-	select {
-	case <-done:
-	case <-time.After(5 * time.Second):
-		t.Fatal("a cancelled context still waited out the pause")
+	s.retryHeld(context.Background())
+	if !held.acked || len(s.parked) != 0 {
+		t.Fatalf("a report past the bound was not let go: %+v, %d held", held, len(s.parked))
 	}
 }