Retire only once the same answer has held ten minutes as well as five passes

Five passes are twenty-five seconds, shorter than a controller restart, a store reconnecting or a file half written; the operator asked for both (hq ADR 0230).
2026-10-06 14:28:21 +02:00
parent e68ef88333
commit 2c15074a0b
10 changed files with 184 additions and 124 deletions
@@ -140,6 +140,9 @@ func namesOf(body map[string]any) string {
 	return strings.Join(out, ",")
 }

+// settle passes every five seconds until the same answer has held for StableFor, and one pass more.
+func (w *world) settle() { w.passes(StableFor + 5*time.Second) }
+
 func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
 	w, said := standingWorld(t)
 	given := holding(w, 1)
@@ -152,33 +155,55 @@ func TestATransientEmptyListForFourPassesRetiresNothing(t *testing.T) {
 	if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
 		t.Fatalf("four passes retired %v and said %v", w.a.removed, retirements(*said))
 	}
-	// Gone again: counted from one, not from where the transient left off.
-	w.give()
-	for i := 0; i < 4; i++ {
-		w.pass()
-	}
-	if len(w.a.removed) != 0 {
-		t.Fatalf("retired after four passes: %v", w.a.removed)
-	}
-	w.pass()
-	if strings.Join(w.a.removed, ",") != "c1" {
-		t.Fatalf("the fifth pass did not retire: %v", w.a.removed)
-	}
 }

-func TestFiveStablePassesRetireAndAskedAgainReEnables(t *testing.T) {
+// Five identical passes are twenty-five seconds — shorter than a controller restart. Both must hold:
+// five passes AND ten minutes of the same answer (novox/hq ADR 0230).
+func TestFivePassesInTwentyFiveSecondsRetireNothingTenMinutesDo(t *testing.T) {
 	w, said := standingWorld(t)
-	given := holding(w, 3)
-	w.give(given[:2]...)
+	holding(w, 1)
+	w.give()
 	for i := 0; i < 5; i++ {
 		w.pass()
 	}
+	if len(w.a.removed) != 0 || len(retirements(*said)) != 0 {
+		t.Fatalf("five passes in 25 s retired %v", w.a.removed)
+	}
+	w.passes(StableFor - 30*time.Second)
+	if len(w.a.removed) != 0 {
+		t.Fatalf("retired before the set held %s: %v", StableFor, w.a.removed)
+	}
+	w.passes(time.Minute)
+	if strings.Join(w.a.removed, ",") != "c1" {
+		t.Fatalf("the same set held %s and was not retired: %v", StableFor, w.a.removed)
+	}
+	// Ten minutes in one slow pass are not five passes.
+	w2 := newWorld(t)
+	holding(w2, 1)
+	w2.give()
+	w2.pass()
+	w2.now = w2.now.Add(StableFor)
+	w2.pass()
+	if len(w2.a.removed) != 0 {
+		t.Fatalf("two passes ten minutes apart retired %v", w2.a.removed)
+	}
+	w2.passes(15 * time.Second)
+	if len(w2.a.removed) != 1 {
+		t.Fatalf("five passes over ten minutes did not retire: %v", w2.a.removed)
+	}
+}
+
+func TestAStableSetIsRetiredAndAskedAgainReEnables(t *testing.T) {
+	w, said := standingWorld(t)
+	given := holding(w, 3)
+	w.give(given[:2]...)
+	w.settle()
 	if strings.Join(w.a.removed, ",") != "c3" {
 		t.Fatalf("retired %v", w.a.removed)
 	}
 	body := lastRetirement(t, *said)
 	if body["change"] != ChangeRetired || namesOf(body) != "c3" || body["held"] != 3 || body["provider-node"] != "anchor" ||
-		!strings.Contains(body["why"].(string), "5 consecutive passes") {
+		!strings.Contains(body["why"].(string), "consecutive passes over 10m") {
 		t.Fatalf("%v", body)
 	}
 	if len(w.a.inv.Retired) != 1 || w.a.inv.Retired[0].Consumer != "c3" {
@@ -203,21 +228,17 @@ func TestAnUnreadablePassStartsTheCountAgain(t *testing.T) {
 	w := newWorld(t)
 	holding(w, 1)
 	w.give()
-	for i := 0; i < 3; i++ {
-		w.pass()
-	}
+	w.passes(StableFor - time.Minute)
 	os.WriteFile(w.receives, []byte("{"), 0o600)
 	w.pass()
 	w.give()
-	for i := 0; i < 4; i++ {
-		w.pass()
-	}
+	w.passes(StableFor - time.Minute)
 	if len(w.a.removed) != 0 {
 		t.Fatalf("an unreadable pass counted: %v", w.a.removed)
 	}
-	w.pass()
+	w.passes(2 * time.Minute)
 	if len(w.a.removed) != 1 {
-		t.Fatalf("not retired after five readable passes: %v", w.a.removed)
+		t.Fatalf("not retired after ten minutes of readable passes: %v", w.a.removed)
 	}
 }

@@ -225,17 +246,13 @@ func TestADifferentSetStartsTheCountAgain(t *testing.T) {
 	w := newWorld(t)
 	given := holding(w, 5)
 	w.give(given[:4]...)
-	for i := 0; i < 3; i++ {
-		w.pass()
-	}
+	w.passes(StableFor - time.Minute)
 	w.give(given[:3]...)
-	for i := 0; i < 4; i++ {
-		w.pass()
-	}
+	w.passes(StableFor - time.Minute)
 	if len(w.a.removed) != 0 {
 		t.Fatalf("a changed set kept its count: %v", w.a.removed)
 	}
-	w.pass()
+	w.passes(2 * time.Minute)
 	if strings.Join(w.a.removed, ",") != "c4,c5" {
 		t.Fatalf("%v", w.a.removed)
 	}
@@ -260,7 +277,7 @@ func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
 	w, said := standingWorld(t)
 	holding(w, 4)
 	w.give()
-	w.passes(10 * time.Minute)
+	w.passes(15 * time.Minute)
 	if len(w.a.removed) != 0 {
 		t.Fatalf("four of four were retired without a person: %v", w.a.removed)
 	}
@@ -275,7 +292,7 @@ func TestTooManyWaitsForAPersonAndApproveRetiresExactlyThatSet(t *testing.T) {
 		t.Fatal("the wait was not said")
 	}
 	// Said again every quarter of an hour while it waits.
-	w.passes(6 * time.Minute)
+	w.passes(11 * time.Minute)
 	if got := strings.Join(retirements(*said), ","); got != "waiting,waiting" {
 		t.Fatalf("%q", got)
 	}
@@ -309,7 +326,7 @@ func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
 	w, said := standingWorld(t)
 	given := holding(w, 4)
 	w.give()
-	w.passes(time.Minute)
+	w.settle()
 	if _, err := w.h.Reject([]string{"c1"}, "no", "operator", ""); err == nil {
 		t.Fatal("rejected a set that is not the one waiting")
 	}
@@ -331,7 +348,11 @@ func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
 	// One of them asked for again: a different answer, so the rejection is settled and counting
 	// starts over — three of four is over the bound again, and waits again.
 	w.give(given[0])
-	w.passes(30 * time.Second)
+	w.pass()
+	if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled" {
+		t.Fatalf("%q", got)
+	}
+	w.settle()
 	if got := strings.Join(retirements(*said), ","); got != "waiting,rejected,settled,waiting" {
 		t.Fatalf("%q", got)
 	}
@@ -339,7 +360,7 @@ func TestRejectedIsKeptAndNotAskedAgainUntilTheSetChanges(t *testing.T) {
 	w2, _ := standingWorld(t)
 	holding(w2, 4)
 	w2.give()
-	w2.passes(time.Minute)
+	w2.settle()
 	w2.h.Reject([]string{"c1", "c2", "c3", "c4"}, "wait", "operator", "")
 	if done, err := w2.h.Approve(ctx, []string{"c1", "c2", "c3", "c4"}, "now", "operator", ""); err != nil || len(done) != 4 {
 		t.Fatal(done, err)
@@ -350,7 +371,7 @@ func TestAWaitingSetAskedForAgainSettles(t *testing.T) {
 	w, said := standingWorld(t)
 	given := holding(w, 4)
 	w.give()
-	w.passes(time.Minute)
+	w.settle()
 	w.give(given...)
 	w.pass()
 	if got := strings.Join(retirements(*said), ","); got != "waiting,settled" || len(w.a.removed) != 0 {
@@ -365,7 +386,7 @@ func TestDeleteRemovesOnlyThatRetiredConsumer(t *testing.T) {
 	w, said := standingWorld(t)
 	given := holding(w, 5)
 	w.give(given[:3]...)
-	w.passes(25 * time.Second)
+	w.settle()
 	if strings.Join(w.a.removed, ",") != "c4,c5" {
 		t.Fatalf("%v", w.a.removed)
 	}
@@ -426,9 +447,7 @@ func TestARestartRetiresWhatTheBackendHoldsUnaskedAndAdoptsWhatItFindsDisabled(t
 	if strings.Join(w.a.removed, ",") != "locked-before" {
 		t.Fatalf("adopting did not mark it: %v", w.a.removed)
 	}
-	for i := 0; i < 5; i++ {
-		w.pass()
-	}
+	w.settle()
 	if strings.Join(w.a.removed, ",") != "locked-before,orphan" {
 		t.Fatalf("an orphan the backend holds was not retired: %v", w.a.removed)
 	}
@@ -444,7 +463,7 @@ func TestABackendThatCannotBeListedIsSaidAndAskedAgain(t *testing.T) {
 	w.a.invErr = nil
 	w.a.inv = Inventory{Active: []string{"c1", "orphan"}}
 	w.pass()
-	w.passes(25 * time.Second)
+	w.settle()
 	if strings.Join(w.a.removed, ",") != "orphan" {
 		t.Fatalf("%v", w.a.removed)
 	}
@@ -454,7 +473,7 @@ func TestTheToolsAnswer(t *testing.T) {
 	w, _ := standingWorld(t)
 	holding(w, 4)
 	w.give()
-	w.passes(time.Minute)
+	w.settle()
 	tools := map[string]func(map[string]any) (any, error){}
 	for _, tool := range RetirementTools(w.h) {
 		tools[tool.Name] = tool.Run
@@ -491,7 +510,7 @@ func TestAFailingConsumerRetiredIsSaidRecovered(t *testing.T) {
 	w.a.failing = errors.New("boom")
 	w.passes(7 * time.Minute)
 	w.give(given[0])
-	w.passes(25 * time.Second)
+	w.settle()
 	recovered := false
 	for _, a := range *said {
 		if a.event == EventRecovered && a.body["consumer"] == "c2" && a.body["why"] == "retired" {