Count the owed effects mesh-delivery tried, so a window's writes can be judged against them

delivery_status said only what is owed now; issue 326's rule needs the tries made in a window.
It now says the tries since this owner started, by how each ended, and the last try's time.
(novox/hq issue 407)

modules/mesh-delivery/cmd/mesh-delivery/effects.go

@@ -284,6 +284,16 @@ func (h *Holder) Flush() {
 			d.Owed = owed(d.Owed)
 			h.keep(d)
 		}
+		// Counted for delivery_status, so a window's writes can be judged against its tries (novox/hq issue 407).
+		h.tried.last = h.Now()
+		switch {
+		case err == nil:
+			h.tried.done++
+		case dropped != "":
+			h.tried.givenUp++
+		default:
+			h.tried.failed++
+		}
 		if dropped != "" {
 			h.refuse(fmt.Errorf("%s: its owed %s given up — %s: %v", j.id, j.e.Kind, dropped, err))
 		}

modules/mesh-delivery/cmd/mesh-delivery/holder.go

@@ -67,6 +67,16 @@ type Holder struct {
 	// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
 	// to stop the walk whole.
 	leaveRefused map[string]string
+	// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
+	tried owedTries
+}
+
+// owedTries is how many owed effects were tried since this owner started, and when the last was: three numbers
+// and a time, however many were tried. Not kept: a restarted owner counts from its start, and says since when, so a
+// window's count is the difference of two readings in the same run (novox/hq issue 407).
+type owedTries struct {
+	done, failed, givenUp int
+	last                  time.Time
 }
 
 // unmatched is a walk no delivery is known for yet: its merge not heard, or merged before this owner held.

modules/mesh-delivery/cmd/mesh-delivery/issue407_test.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+// triedSaid is delivery_status's count of owed effects tried (novox/hq issue 407), read as a test reads it.
+func triedSaid(t *testing.T, w *world) map[string]any {
+	t.Helper()
+	got, ok := w.h.Status("listening")["owed-tried"].(map[string]any)
+	if !ok {
+		t.Fatalf("delivery_status says no owed-tried: %v", w.h.Status("listening"))
+	}
+	return got
+}
+
+func countOf(t *testing.T, m map[string]any, key string) int {
+	t.Helper()
+	n, ok := m[key].(int)
+	if !ok {
+		t.Fatalf("owed-tried has no count %q: %v", key, m)
+	}
+	return n
+}
+
+// novox/hq issue 407: delivery_status says how many owed effects were tried since this owner started, how
+// many were done, failed to be tried again, or were given up, and when the last was tried — so a window's
+// writes can be checked against the tries made in it (issue 326's rule) by reading it at both ends.
+func TestDeliveryStatusCountsTheOwedEffectsTried(t *testing.T) {
+	w := newWorld(t)
+	start := w.now
+	tried := triedSaid(t, w)
+	if countOf(t, tried, "tried") != 0 || countOf(t, tried, "done") != 0 || countOf(t, tried, "failed") != 0 ||
+		countOf(t, tried, "given-up") != 0 {
+		t.Fatalf("a holder that tried nothing says %v", tried)
+	}
+	if tried["since"] != start.Format(time.RFC3339) {
+		t.Fatalf("its count is since %v, the holder started %s", tried["since"], start.Format(time.RFC3339))
+	}
+	if _, there := tried["last"]; there {
+		t.Fatalf("nothing tried, yet a last try is said: %v", tried)
+	}
+
+	// A head seen: its event and its note are owed and done.
+	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
+	w.settleAll()
+	tried = triedSaid(t, w)
+	done := countOf(t, tried, "done")
+	if done < len(w.said)+len(w.forge.notesOn(head)) || done == 0 {
+		t.Fatalf("said %d and noted %d, yet done counts %d", len(w.said), len(w.forge.notesOn(head)), done)
+	}
+	if countOf(t, tried, "tried") != done || countOf(t, tried, "failed") != 0 {
+		t.Fatalf("every try succeeded, yet %v", tried)
+	}
+	if tried["last"] != w.now.Format(time.RFC3339) {
+		t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339))
+	}
+
+	// The forge away: its effects fail and are counted failed; quiet passes try nothing more.
+	w.forge.down = true
+	w.later(time.Minute)
+	w.h.Checked(verdict("novox/app", 7, head, "pass"))
+	w.h.Flush()
+	tried = triedSaid(t, w)
+	failed := countOf(t, tried, "failed")
+	if failed == 0 {
+		t.Fatalf("the forge away failed nothing: %v", tried)
+	}
+	if countOf(t, tried, "tried") != countOf(t, tried, "done")+failed+countOf(t, tried, "given-up") {
+		t.Fatalf("tries do not add up: %v", tried)
+	}
+	if tried["last"] != w.now.Format(time.RFC3339) {
+		t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339))
+	}
+	before := countOf(t, tried, "tried")
+	w.h.Flush() // before each failure's next try: nothing is tried
+	if n := countOf(t, triedSaid(t, w), "tried"); n != before {
+		t.Fatalf("a pass before any next try tried %d", n-before)
+	}
+
+	// The forge back, past every next try: the failed effects are done.
+	w.forge.down = false
+	w.later(time.Hour)
+	w.h.Flush()
+	tried = triedSaid(t, w)
+	if countOf(t, tried, "tried") != before+failed || countOf(t, tried, "done") <= done {
+		t.Fatalf("the forge back: tried %d (was %d, %d failed), %v", countOf(t, tried, "tried"), before, failed, tried)
+	}
+}
+
+// The count is this run's: a restarted holder says it counts since its own start, from nothing.
+func TestOwedTriedCountsSinceTheStart(t *testing.T) {
+	w := newWorld(t)
+	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
+	w.settleAll()
+	if countOf(t, triedSaid(t, w), "tried") == 0 {
+		t.Fatal("nothing counted before the restart")
+	}
+	again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
+	tried := triedSaid(t, again)
+	if countOf(t, tried, "tried") != 0 || tried["since"] != again.now.Format(time.RFC3339) {
+		t.Fatalf("a restarted holder says %v", tried)
+	}
+}

modules/mesh-delivery/cmd/mesh-delivery/main.go

@@ -347,7 +347,8 @@ func tools(h *Holder, l *listening) []stdio.Tool {
 			}},
 		{Name: "delivery_status",
 			Description: "Whether this owner works: how many deliveries in each state, the groups, what is owed to " +
-				"the forge and the bus and not yet done, walks with no delivery yet, whether its events arrive, when " +
+				"the forge and the bus and not yet done, how many owed effects were tried since it started (done, failed " +
+				"and tried again, given up) and when the last was, walks with no delivery yet, whether its events arrive, when " +
 				"the controller's walks were last read or why they could not be, and the last refusals.",
 			Run: func(map[string]any) (any, error) { return h.Status(l.get()), nil }},
 	}

modules/mesh-delivery/cmd/mesh-delivery/verbs.go

@@ -648,6 +648,15 @@ func (h *Holder) Status(listening string) map[string]any {
 	out := map[string]any{"deliveries": byState, "groups": len(h.groups), "owed": owed, "unkept": len(h.dirty),
 		"walks-unmatched": len(h.unmatched), "events": listening, "started": h.started.Format(time.RFC3339),
 		"seat-on-record": h.held}
+	// The owed effects tried since this owner started (novox/hq issue 407): done, failed and to be tried again,
+	// or given up, and when the last was tried. A count since the start, said with that start: two readings in
+	// one run give a window's tries.
+	tried := map[string]any{"since": h.started.Format(time.RFC3339), "done": h.tried.done, "failed": h.tried.failed,
+		"given-up": h.tried.givenUp, "tried": h.tried.done + h.tried.failed + h.tried.givenUp}
+	if !h.tried.last.IsZero() {
+		tried["last"] = h.tried.last.Format(time.RFC3339)
+	}
+	out["owed-tried"] = tried
 	// Each held delivery by name, with why and the verb that releases it (novox/hq issue 529): a count alone
 	// pointed a reader at nothing.
 	var held []string