A failure that repeats is said to be stuck

The mesh kept one report per machine, replaced, so a resource nothing can ever apply looked like a failure that had just happened, every reconcile interval, for ever. The row now keeps when the current failure began and how many reports in a row have said it — the same outcome, refusal and failed resources; anything different starts again and a clean apply clears it. Three make the machine stuck, and status says so beside the failure, in words and in JSON (novox/hq 04-ISSUES/065, ADR 0090).
2026-09-21 17:43:24 +02:00
parent 69d0b94395
commit fcaad7271a
6 changed files with 183 additions and 8 deletions
@@ -245,3 +245,79 @@ func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) {
 		t.Fatalf("a machine the caller could not work out was reported as waiting: %+v", waiting)
 	}
 }
+
+// **A failure that repeats is told apart from one that just happened.** A node re-applies on a
+// steady interval and reports each time, so a resource that will never apply arrives as the same
+// report over and over — one row, replaced, "failed" at a fresh time — and nothing distinguished
+// it from a failure that goes away by itself (novox/hq 04-ISSUES/065). The row now keeps when the
+// current failure began and how many reports in a row have said it.
+func TestTheSameFailureReportedAgainIsCountedNotRestarted(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	id := nodeNamed(t, inv, "looping")
+	same := Doing{Outcome: OutcomeFailed, Failed: []FailedResource{{ID: "img", Error: "no such image"}}}
+
+	if err := inv.RecordDoing(ctx, id, same); err != nil {
+		t.Fatal(err)
+	}
+	first, _, err := inv.DoingOf(ctx, "looping")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if first.Times != 1 || first.Since == nil || first.Stuck() {
+		t.Fatalf("one failure is one failure, not yet stuck: %+v", first)
+	}
+
+	for range StuckAfter - 1 {
+		if err := inv.RecordDoing(ctx, id, same); err != nil {
+			t.Fatal(err)
+		}
+	}
+	again, _, err := inv.DoingOf(ctx, "looping")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again.Times != StuckAfter || !again.Stuck() {
+		t.Fatalf("the same failure %d times is stuck: %+v", StuckAfter, again)
+	}
+	if !again.Since.Equal(*first.Since) {
+		t.Fatalf("the failure began at %s and the row now says %s", *first.Since, *again.Since)
+	}
+
+	// A different failure is a new situation, not a longer one.
+	other := Doing{Outcome: OutcomeFailed, Failed: []FailedResource{{ID: "svc", Error: "unit not found"}}}
+	if err := inv.RecordDoing(ctx, id, other); err != nil {
+		t.Fatal(err)
+	}
+	changed, _, err := inv.DoingOf(ctx, "looping")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if changed.Times != 1 || changed.Stuck() || !changed.Since.After(*first.Since) && !changed.Since.Equal(*first.Since) {
+		t.Fatalf("a new failure starts the count again: %+v", changed)
+	}
+
+	// And a clean apply clears it: the machine is doing what it was told, since nothing.
+	if err := inv.RecordDoing(ctx, id, Doing{Outcome: OutcomeApplied, Applied: 2}); err != nil {
+		t.Fatal(err)
+	}
+	fine, _, err := inv.DoingOf(ctx, "looping")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if fine.Times != 0 || fine.Since != nil || fine.Stuck() {
+		t.Fatalf("a machine doing what it was told is not stuck: %+v", fine)
+	}
+
+	// The list of what is wrong carries the count, so `status` can say it.
+	if err := inv.RecordDoing(ctx, id, same); err != nil {
+		t.Fatal(err)
+	}
+	wrong, err := inv.NotDoingWhatTheyWereTold(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wrong) != 1 || wrong[0].Times != 1 || wrong[0].Since == nil {
+		t.Fatalf("got %+v", wrong)
+	}
+}