Merge pull request 'mesh-delivery: retire history instead of holding it; end owed work the forge cannot do (hq issue 313)' (#121) from fix/313-retire-adopted-history into main

modules/mesh-delivery/cmd/mesh-delivery/effects.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -104,6 +105,8 @@ func statusOf(d *Delivery) (string, string) {
 		return "warning", clip("superseded: "+why, 140)
 	case Stopped:
 		return "error", clip("stopped: "+why, 140)
+	case Retired:
+		return "success", "history: merged before this owner delivered anything, never walked"
 	}
 	return "pending", string(d.State)
 }
@@ -186,6 +189,7 @@ func (h *Holder) Flush() {
 		url   string
 	}
 	h.mu.Lock()
+	now := h.Now()
 	for id := range h.dirty {
 		if d := h.deliveries[id]; d != nil {
 			h.keep(d)
@@ -208,6 +212,9 @@ func (h *Holder) Flush() {
 			continue
 		}
 		for _, e := range d.Owed {
+			if e.Next != nil && now.Before(*e.Next) {
+				continue
+			}
 			j := job{id: id, e: e, repo: [2]string{d.Owner(), d.Repo()}, num: d.Number, url: d.HTMLURL}
 			if e.Kind == EffectView {
 				j.body = ViewBody(d, h.groups[d.Group])
@@ -220,6 +227,9 @@ func (h *Holder) Flush() {
 			continue
 		}
 		for _, e := range g.Owed {
+			if e.Next != nil && now.Before(*e.Next) {
+				continue
+			}
 			jobs = append(jobs, job{id: id, group: true, e: e})
 		}
 	}
@@ -228,15 +238,26 @@ func (h *Holder) Flush() {
 	for _, j := range jobs {
 		err := h.do(j.e, j.repo, j.num, j.body, j.url, j.group)
 		h.mu.Lock()
+		// Why it is given up, when it is: the forge's word that what it is for does not exist is final, and
+		// so is a failure tried retryLimit times, the last ones retryCap apart.
+		dropped := ""
+		switch {
+		case err != nil && forgeKinds[j.e.Kind] && doesNotExist(err):
+			dropped = "the forge answers that what it is for does not exist"
+		case err != nil && j.e.Tries+1 >= retryLimit:
+			dropped = fmt.Sprintf("tried %d times since %s", j.e.Tries+1, j.e.Since.UTC().Format(time.RFC3339))
+		}
 		owed := func(list []Effect) []Effect {
 			for i, o := range list {
 				if o.Kind == j.e.Kind && o.Commit == j.e.Commit && o.Context == j.e.Context && o.Line == j.e.Line &&
 					o.Since.Equal(j.e.Since) {
-					if err == nil {
+					if err == nil || dropped != "" {
 						return append(list[:i:i], list[i+1:]...)
 					}
 					list[i].Tries++
 					list[i].Last = err.Error()
+					next := h.Now().Add(retryAfter(list[i].Tries))
+					list[i].Next = &next
 					return list
 				}
 			}
@@ -251,13 +272,43 @@ func (h *Holder) Flush() {
 			d.Owed = owed(d.Owed)
 			h.keep(d)
 		}
+		if dropped != "" {
+			h.refuse(fmt.Errorf("%s: its owed %s given up — %s: %v", j.id, j.e.Kind, dropped, err))
+		}
 		h.mu.Unlock()
-		if err != nil && j.e.Tries%10 == 0 {
-			h.Logf("[mesh-delivery] %s: %s not done yet (%v); tried again", j.id, j.e.Kind, err)
+		if err != nil && dropped == "" && j.e.Tries%10 == 0 {
+			h.Logf("[mesh-delivery] %s: %s not done yet (%v); tried again in %s", j.id, j.e.Kind, err,
+				retryAfter(j.e.Tries+1))
 		}
 	}
 }
 
+// An owed effect that fails is tried again later each time: retryFirst after the first failure, doubling up
+// to retryCap, and given up after retryLimit tries — a little over two days — saying so (novox/hq issue 313).
+const (
+	retryFirst = 10 * time.Second
+	retryCap   = time.Hour
+	retryLimit = 60
+)
+
+// retryAfter is how long after its tries-th failure an effect is tried again.
+func retryAfter(tries int) time.Duration {
+	wait := retryFirst
+	for i := 1; i < tries && wait < retryCap; i++ {
+		wait *= 2
+	}
+	return min(wait, retryCap)
+}
+
+// forgeKinds are the effects done by the forge's holder.
+var forgeKinds = map[string]bool{EffectNote: true, EffectView: true, EffectStatus: true}
+
+// notThere is the forge's own answer that what was asked of does not exist: a pull request whose number the
+// forge no longer holds, a commit it does not know. Tried again it answers the same, for ever.
+var notThere = regexp.MustCompile(`(?i)Gitea API \S+: (404\b|\d{3}\b.*\bdoes not exist\b)`)
+
+func doesNotExist(err error) bool { return err != nil && notThere.MatchString(err.Error()) }
+
 // do is one owed effect.
 func (h *Holder) do(e Effect, repo [2]string, number int, body, url string, group bool) error {
 	switch e.Kind {

modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go

@@ -220,6 +220,9 @@ type fakeForge struct {
 	views    map[string]string
 	statuses map[string]string
 	down     bool
+	// viewErr is what View answers instead, when set: a pull request the forge no longer holds.
+	viewErr   error
+	viewCalls int
 	// What Statuses reads: the pull requests by owner/repo#number, the base's required statuses, whether the
 	// protection cannot be read, and what it was asked.
 	pulls          map[string]ForgePull
@@ -251,6 +254,10 @@ func (f *fakeForge) Note(owner, repo, commit, line string) error {
 func (f *fakeForge) View(owner, repo string, number int, body string) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	f.viewCalls++
+	if f.viewErr != nil {
+		return f.viewErr
+	}
 	if f.down {
 		return errors.New("the forge is away")
 	}

modules/mesh-delivery/cmd/mesh-delivery/history_test.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+// History (novox/hq issue 313): merges long past, heard only from the forge's word when this owner first held
+// the seat, reached the machines before it existed. They are retired — never held for a person, released or
+// delivered — and what is owed for a pull request the forge no longer holds is given up, not tried for ever.
+
+// aPastMerge is the forge's `pull.merged` replayed from the bus's history: merged days before it is heard.
+func aPastMerge(repo string, number int, head, merge string, at time.Time) PullEvent {
+	p := merged(repo, number, head, merge)
+	p.MergedAt = at.UTC().Format(time.RFC3339)
+	return p
+}
+
+func TestAMergeLongPastHeardFromTheForgesWordIsRetiredAtAdoption(t *testing.T) {
+	w := newWorld(t)
+	id := IDOf("novox/app", head)
+	w.h.PullMerged(aPastMerge("novox/app", 274, head, merge, w.now.Add(-4*24*time.Hour)))
+	w.settleAll()
+	d := w.delivery(id)
+	if d.State != Retired || !strings.Contains(d.History, "history") {
+		t.Fatalf("a merge four days past heard from the forge's word is %s (%q)", d.State, d.History)
+	}
+	if got := statesOf([]State{d.Transitions[0].To, d.Transitions[1].To}); strings.Join(got, ",") != "proposed,retired" ||
+		d.Transitions[1].Event != EvHistory {
+		t.Fatalf("transitions %+v", d.Transitions)
+	}
+	// Nothing is said on its pull request — the forge may hold another under its number — and nothing stalls.
+	if w.forge.viewCalls != 0 || len(w.forge.statuses) != 0 {
+		t.Fatalf("history was shown on a pull request: %d view(s), statuses %v", w.forge.viewCalls, w.forge.statuses)
+	}
+	if len(d.Owed) != 0 {
+		t.Fatalf("still owed: %+v", d.Owed)
+	}
+	if s := w.h.Stalled(); len(s) != 0 {
+		t.Fatalf("history stalls: %+v", s)
+	}
+	// Its record is still kept, on the commit's note.
+	if len(w.forge.notesOn(head)) == 0 {
+		t.Fatal("no note of it on its commit")
+	}
+
+	// A merge heard from the forge's word within moments of happening is a live one: held for a person.
+	live := IDOf("novox/app", "bbbbbbbbbbbb2222")
+	w.h.PullMerged(aPastMerge("novox/app", 9, "bbbbbbbbbbbb2222", "cccccccccccc3333", w.now.Add(-time.Minute)))
+	if s := w.state(live); s != Held {
+		t.Fatalf("a live merge from the forge's word is %s", s)
+	}
+	// A merge long past whose walk the controller still holds open is delivered by it, never history.
+	walked := IDOf("novox/app", "dddddddddddd4444")
+	w.h.WalkMoved(aWalk("plan-9", "novox/app", "eeeeeeeeeeee5555", true))
+	w.h.PullMerged(aPastMerge("novox/app", 10, "dddddddddddd4444", "eeeeeeeeeeee5555", w.now.Add(-48*time.Hour)))
+	if s := w.state(walked); s == Retired {
+		t.Fatal("a merge with an open walk was retired as history")
+	}
+	// A merge the forge announces under a number an older delivery already merged with is not that delivery.
+	w.h.PullMerged(aPastMerge("novox/app", 274, "", "ffffffffffff6666", w.now.Add(-time.Minute)))
+	if d := w.delivery(id); d.MergedAs != merge {
+		t.Fatalf("a reused number merged the old delivery again as %s", d.MergedAs)
+	}
+}
+
+// oldHistory is a held delivery as the first adoption left it: made from the forge's word, held, owing a view
+// on a pull request the forge no longer holds.
+func oldHistory(id, repo string, number int, created, mergedAt time.Time) *Delivery {
+	commit := strings.SplitN(id, "@", 2)[1] + "0000"
+	return &Delivery{ID: id, Repository: repo, Commit: commit, Number: number, Base: "main", MergedAs: commit,
+		MergedAt: mergedAt, Created: created, Since: created, State: Held,
+		HeldWhy: "merged without a passing check: only a person decides that it goes on",
+		Transitions: []Transition{
+			{At: created, To: Proposed, Event: EvAnnounced, Why: repo + "#274 merged; its head was not heard by this " +
+				"owner, so it is made from the forge's word"},
+			{At: created, From: Proposed, To: Held, Event: EvMergedUnchecked, Why: "merged without a passing check"}},
+		Owed: []Effect{{Kind: EffectView, Since: created, Tries: 11051,
+			Last: "Gitea API /repos/" + repo + "/issues/274/comments: 500 issue does not exist"}}}
+}
+
+func TestRetireHistoryRetiresTheAdoptedHistoryInBulkAndLeavesEveryOtherHeldOne(t *testing.T) {
+	store := newMemStore()
+	start := time.Date(2026, 10, 6, 23, 5, 27, 0, time.UTC)
+	for _, d := range []*Delivery{
+		oldHistory("novox/hq@018ee359ae52", "novox/hq", 274, start, start.Add(-4*24*time.Hour)),
+		oldHistory("novox/lab@a4f6bb8ce054", "novox/lab", 53, start, start.Add(-2*time.Hour)),
+		// Made from the forge's word, but heard a minute after its merge: a live merge, a person's.
+		oldHistory("novox/hq@7777777777aa", "novox/hq", 3, start, start.Add(-time.Minute)),
+	} {
+		if err := store.PutDelivery(d); err != nil {
+			t.Fatal(err)
+		}
+	}
+	w := newWorldOver(t, store, newFakeController(), newFakeForge(), start.Add(36*time.Hour))
+	// And one heard live: rejected, then merged — a person's to decide.
+	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
+	w.h.Checked(verdict("novox/app", 7, head, "fail"))
+	w.h.PullMerged(merged("novox/app", 7, head, merge))
+	if len(w.h.Stalled()) != 3 { // the three adopted at the switch; the one held just now is within its bound
+		t.Fatalf("stalled before: %+v", w.h.Stalled())
+	}
+
+	dry, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, true)
+	if err != nil || dry.Retired != 2 || dry.LeftHeld != 2 || len(dry.Left) != 2 {
+		t.Fatalf("a dry retire: %+v %v", dry, err)
+	}
+	if s := w.state("novox/hq@018ee359ae52"); s != Held {
+		t.Fatalf("a dry retire moved a delivery to %s", s)
+	}
+	if _, err := w.h.RetireHistory("", "", "", aPerson, false); err == nil {
+		t.Fatal("a retire with no why was taken")
+	}
+
+	got, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, false)
+	if err != nil || got.Retired != 2 || got.ByRepository["novox/hq"] != 1 || got.ByRepository["novox/lab"] != 1 {
+		t.Fatalf("retire: %+v %v", got, err)
+	}
+	for _, id := range []string{"novox/hq@018ee359ae52", "novox/lab@a4f6bb8ce054"} {
+		d := w.delivery(id)
+		if d.State != Retired || d.Transitions[len(d.Transitions)-1].By != aPerson {
+			t.Fatalf("%s is %s", id, d.State)
+		}
+		for _, e := range d.Owed {
+			if e.Kind == EffectView || e.Kind == EffectStatus {
+				t.Fatalf("%s still owes its pull request: %+v", id, e)
+			}
+		}
+	}
+	for _, id := range []string{"novox/hq@7777777777aa", IDOf("novox/app", head)} {
+		if s := w.state(id); s != Held {
+			t.Fatalf("%s, not history, is %s", id, s)
+		}
+	}
+	// Nothing was released, delivered or let go.
+	if len(w.ctl.delivers) != 0 || len(w.ctl.stops) != 0 {
+		t.Fatalf("a retire asked the controller: deliver %v stop %v", w.ctl.delivers, w.ctl.stops)
+	}
+	if s := w.h.Stalled(); len(s) != 1 || s[0].ID != "novox/hq@7777777777aa" {
+		t.Fatalf("stalled after: %+v", s)
+	}
+	// Retired is final: a second retire takes nothing, and nothing leaves it.
+	again, _ := w.h.RetireHistory("", "", "again", aPerson, false)
+	if again.Retired != 0 {
+		t.Fatalf("retired twice: %+v", again)
+	}
+	if _, err := w.h.Release("novox/hq@018ee359ae52", "go on", aPerson); err == nil {
+		t.Fatal("a retired delivery was released")
+	}
+}
+
+func TestRetireHistoryIsRefusedForARealHeldDelivery(t *testing.T) {
+	w := newWorld(t)
+	id := IDOf("novox/app", head)
+	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
+	w.h.Checked(verdict("novox/app", 7, head, "fail"))
+	w.h.PullMerged(merged("novox/app", 7, head, merge))
+	w.later(48 * time.Hour)
+	_, err := w.h.RetireHistory(id, "", "looks old", aPerson, false)
+	var refused ErrRefused
+	if !errors.As(err, &refused) || !strings.Contains(err.Error(), "heard its head") {
+		t.Fatalf("a held delivery whose head was heard was retired: %v", err)
+	}
+	if s := w.state(id); s != Held {
+		t.Fatalf("a refused retire left it %s", s)
+	}
+	// A delivery that is not held is refused too, by the table.
+	other := IDOf("novox/app", "bbbbbbbbbbbb2222")
+	w.h.PullUpdated(pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y", "src/b.go"))
+	if _, err := w.h.RetireHistory(other, "", "old", aPerson, false); err == nil || w.state(other) != Proposed {
+		t.Fatalf("a proposed delivery was retired: %v", err)
+	}
+	if _, err := w.h.RetireHistory("novox/app@nothing", "", "old", aPerson, false); err == nil {
+		t.Fatal("retired a delivery that does not exist")
+	}
+}
+
+func TestOwedWorkTheForgeSaysCanNeverBeDoneIsDropped(t *testing.T) {
+	w := newWorld(t)
+	id := IDOf("novox/app", head)
+	w.forge.viewErr = errors.New(`Gitea API /repos/novox/app/issues/7/comments: 500 {"message":"issue does not ` +
+		`exist [id: 0, repo_id: 53, index: 7]"}`)
+	w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
+	w.h.Flush()
+	if w.forge.viewCalls != 1 {
+		t.Fatalf("the view was asked %d time(s)", w.forge.viewCalls)
+	}
+	if d := w.delivery(id); len(d.Owed) != 0 {
+		t.Fatalf("an answer that its pull request does not exist left owed %+v", d.Owed)
+	}
+	w.settleAll()
+	if w.forge.viewCalls != 1 {
+		t.Fatalf("a dropped view was asked again: %d", w.forge.viewCalls)
+	}
+	if st := w.h.Status("listening"); !strings.Contains(strings.Join(st["refused"].([]string), "\n"), "given up") {
+		t.Fatalf("its drop was not said: %v", st["refused"])
+	}
+
+	// Any other failure is tried again later each time, and given up after retryLimit tries.
+	w.forge.viewErr = errors.New("the forge is away")
+	w.h.Checked(verdict("novox/app", 7, head, "pass"))
+	before := w.forge.viewCalls
+	w.h.Flush()
+	w.h.Flush()
+	w.h.Flush()
+	if w.forge.viewCalls != before+1 {
+		t.Fatalf("a failed view was tried %d time(s) at once", w.forge.viewCalls-before)
+	}
+	w.later(retryFirst + time.Second)
+	w.h.Flush()
+	if w.forge.viewCalls != before+2 {
+		t.Fatalf("a failed view was not tried again after its wait: %d", w.forge.viewCalls-before)
+	}
+	if retryAfter(1) != retryFirst || retryAfter(2) != 2*retryFirst || retryAfter(100) != retryCap {
+		t.Fatalf("the waits: %s %s %s", retryAfter(1), retryAfter(2), retryAfter(100))
+	}
+	for range retryLimit {
+		w.later(retryCap)
+		w.h.Flush()
+	}
+	for _, e := range w.delivery(id).Owed {
+		if e.Kind == EffectView {
+			t.Fatalf("a view failing %d times is still owed: %+v", e.Tries, e)
+		}
+	}
+	if !doesNotExist(errors.New("Gitea API /repos/a/b/issues/1/comments: 404 Not Found")) ||
+		doesNotExist(errors.New("tool gitea.gitea_delivery_view does not exist on this machine")) ||
+		doesNotExist(errors.New("the forge is away")) {
+		t.Fatal("doesNotExist reads the forge's answer wrong")
+	}
+}

modules/mesh-delivery/cmd/mesh-delivery/holder.go

@@ -138,7 +138,7 @@ func (h *Holder) moved(d *Delivery, ts []Transition) {
 			d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: executedLine(d), Since: now})
 		}
 	}
-	if len(ts) > 0 && d.Number > 0 {
+	if len(ts) > 0 && d.shownOnItsPullRequest() {
 		d.owe(Effect{Kind: EffectView, Since: now})
 		state, desc := statusOf(d)
 		d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
@@ -159,7 +159,8 @@ func (d *Delivery) owe(e Effect) {
 }
 
 func afterTheTrunk(s State) bool {
-	return s == Published || s == Delivering || s == Held || s == Delivered || s == Failed || s == Stopped || s == Superseded
+	return s == Published || s == Delivering || s == Held || s == Delivered || s == Failed || s == Stopped ||
+		s == Superseded || s == Retired
 }
 
 func orNothing(s State) string {
@@ -357,12 +358,15 @@ func (h *Holder) PullMerged(p PullEvent) {
 	if d == nil {
 		var newest time.Time
 		for _, c := range h.deliveries {
-			if c.Repository == p.repository() && c.Number == p.Number && c.NewerHead == "" && c.Created.After(newest) {
+			// One already on the trunk is not merged again: a forge whose numbers began again reuses them.
+			if c.Repository == p.repository() && c.Number == p.Number && c.NewerHead == "" && c.MergedAs == "" &&
+				c.Created.After(newest) {
 				d, newest = c, c.Created
 			}
 		}
 	}
 	now := h.Now()
+	var ts []Transition
 	if d == nil {
 		head := p.HeadSHA
 		if head == "" {
@@ -371,7 +375,7 @@ func (h *Holder) PullMerged(p PullEvent) {
 		d = &Delivery{ID: IDOf(p.repository(), head), Repository: p.repository(), Commit: head, Number: p.Number,
 			Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
 			PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
-			After: AfterIn(p.Body), Created: now}
+			After: AfterIn(p.Body), Created: now, FromForge: true}
 		// Its check, as the forge holds it on the head: the verdict the pull request merged with.
 		if s := p.HeadChecks["mesh/merge-gate"]; s != "" {
 			d.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it",
@@ -383,7 +387,7 @@ func (h *Holder) PullMerged(p PullEvent) {
 			h.refuse(err)
 			return
 		}
-		h.moved(d, []Transition{t})
+		ts = append(ts, t)
 	}
 	d.MergedAs = p.MergeCommit
 	if t, err := time.Parse(time.RFC3339, p.MergedAt); err == nil {
@@ -398,7 +402,14 @@ func (h *Holder) PullMerged(p PullEvent) {
 			delete(h.unmatched, id)
 		}
 	}
-	ts := settle(d, Facts{Now: now})
+	// A merge long past heard only now, from the forge's word — replayed from the bus's history when this owner
+	// first held the seat — is history: it reached the machines before this owner was there to deliver it, and
+	// holding it for a person asks a person about the past (novox/hq issue 313).
+	if d.FromForge && d.History == "" && isHistory(d) == nil {
+		d.History = fmt.Sprintf("merged %s, %s before this owner heard of it from the forge's word: history, "+
+			"never to be walked", d.MergedAt.UTC().Format(time.RFC3339), now.Sub(d.MergedAt).Round(time.Minute))
+	}
+	ts = append(ts, settle(d, Facts{Now: now})...)
 	if d.State == Held && d.HeldWhy == "" {
 		d.HeldWhy = "merged without a passing check: only a person decides that it goes on"
 	}
@@ -1020,7 +1031,7 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() {
 	for i, m := range ordered {
 		d := byID[m.ID]
 		switch {
-		case d.State == Delivered:
+		case d.State == Delivered || d.State == Retired:
 			continue
 		case d.State == Failed || d.State == Stopped || d.State == Superseded:
 			var asks []func()
@@ -1082,7 +1093,7 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
 		return "failed", "a member failed"
 	case count[Stopped] > 0 && g.Closed:
 		return "stopped", "a member was stopped"
-	case n > 0 && count[Delivered] == n:
+	case n > 0 && count[Delivered]+count[Retired] == n:
 		return "delivered", "every member is delivered"
 	case g.Closed:
 		return "delivering", fmt.Sprintf("%d of %d delivered", count[Delivered], n)

modules/mesh-delivery/cmd/mesh-delivery/holder_test.go

@@ -256,6 +256,7 @@ func TestATransitionIsKeptBeforeItIsSaid(t *testing.T) {
 		t.Fatal("noted while the forge was away")
 	}
 	w.forge.down = false
+	w.later(time.Hour) // each failure put its next try later: past every one of them
 	w.settleAll()
 	w.settleAll()
 	if notes := w.forge.notesOn(head); len(notes) != 3 {

modules/mesh-delivery/cmd/mesh-delivery/main.go

@@ -290,6 +290,25 @@ func tools(h *Holder, l *listening) []stdio.Tool {
 				}
 				return h.Close(strArg(a, "id"), strArg(a, "why"))
 			}},
+		{Name: seat + "retire-history",
+			Description: "Retire held deliveries that are history, on a person's word, with why: each made from the " +
+				"forge's word for a merge on the trunk heard 15 minutes or more after it happened — replayed when this " +
+				"owner first held the seat — with no walk of it open. Never released, delivered or put back: retired, " +
+				"final. One by id, refused when the rule does not take it; or every held one (of one repository when " +
+				"named), answering how many it retired and naming those it left held. dry \"true\" answers what it " +
+				"would do and changes nothing.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{
+				"why": str("why, kept with every transition"), "id": str("one delivery's id; every held one when empty"),
+				"repository": str("owner/repository: only its held deliveries"),
+				"dry":        map[string]any{"type": "string", "enum": []string{"true", "false"}}},
+				"required": []string{"why"}},
+			Run: func(a map[string]any) (any, error) {
+				if err := need(a, "why"); err != nil {
+					return nil, err
+				}
+				return h.RetireHistory(strArg(a, "id"), strArg(a, "repository"), strArg(a, "why"), aPerson,
+					boolArg(a, "dry"))
+			}},
 		{Name: "delivery_status",
 			Description: "Whether this owner works: how many deliveries in each state, the groups, what is owed to " +
 				"the forge and the bus and not yet done, walks with no delivery yet, whether its events arrive, when " +

modules/mesh-delivery/cmd/mesh-delivery/model.go

@@ -57,6 +57,10 @@ type Delivery struct {
 	MergedAs       string    `json:"merged_as,omitempty"`
 	MergedAt       time.Time `json:"merged_at"`
 	HeldWhy        string    `json:"held_why,omitempty"`
+	// FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner.
+	// History is why it is history, once it is (novox/hq issue 313).
+	FromForge bool   `json:"from_forge,omitempty"`
+	History   string `json:"history,omitempty"`
 	// Walk is the controller's walk of its trunk commit, as last read; Steps the machines it reached.
 	Walk  *WalkSeen `json:"walk,omitempty"`
 	Steps []Step    `json:"steps,omitempty"`
@@ -167,6 +171,8 @@ type Effect struct {
 	Tries   int       `json:"tries,omitempty"`
 	Last    string    `json:"last,omitempty"`
 	Since   time.Time `json:"since"`
+	// Next is when a failed effect is tried again: later after every failure, up to retryCap.
+	Next *time.Time `json:"next,omitempty"`
 }
 
 // The kinds of effect.
@@ -177,6 +183,22 @@ const (
 	EffectStatus = "status"
 )
 
+// FromForgesWord is whether the delivery was made from the forge's word at its merge: said by FromForge, or —
+// for one made before FromForge was kept — by its first transition.
+func (d *Delivery) FromForgesWord() bool {
+	if d.FromForge {
+		return true
+	}
+	return len(d.Transitions) > 0 && d.Transitions[0].Event == EvAnnounced &&
+		strings.Contains(d.Transitions[0].Why, "so it is made from the forge's word")
+}
+
+// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
+// pull request the forge may no longer hold — or holds under its number another one.
+func (d *Delivery) shownOnItsPullRequest() bool {
+	return d.Number > 0 && d.History == "" && d.State != Retired
+}
+
 // Owner and Repo split the repository.
 func (d *Delivery) Owner() string { o, _, _ := strings.Cut(d.Repository, "/"); return o }
 func (d *Delivery) Repo() string  { _, r, _ := strings.Cut(d.Repository, "/"); return r }

modules/mesh-delivery/cmd/mesh-delivery/table.go

@@ -30,15 +30,19 @@ const (
 	Failed     State = "failed"
 	Superseded State = "superseded"
 	Stopped    State = "stopped"
+	// Retired is history: a merge this owner heard only from the forge's word, long after it happened —
+	// replayed from the bus when it first held the seat. It reached the machines before this owner existed,
+	// and nothing is delivered, released or put back for it (novox/hq issue 313).
+	Retired State = "retired"
 )
 
 // AllStates are every state a delivery can be in, for the table's own test and its verb.
 var AllStates = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held, Delivered, Failed,
-	Superseded, Stopped}
+	Superseded, Stopped, Retired}
 
 // Final is whether nothing follows a state.
 func (s State) Final() bool {
-	return s == Delivered || s == Failed || s == Superseded || s == Stopped
+	return s == Delivered || s == Failed || s == Superseded || s == Stopped || s == Retired
 }
 
 // Event is what moves a delivery.
@@ -66,6 +70,8 @@ const (
 	EvFailed          Event = "failed"           // its walk failed: a gate, a build, a machine
 	EvSuperseded      Event = "superseded"       // a newer delivery to the same trunk took over its walk
 	EvStop            Event = "stop"             // a person stopped it, or its group did
+	EvHistory         Event = "history"          // made from the forge's word for a merge long past: history
+	EvRetire          Event = "retire"           // a person's word that a held delivery is history, never to be walked
 )
 
 // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act.
@@ -114,6 +120,27 @@ var Table = []Row{
 			return unless(d.Walk != nil && d.Walk.Started(), "no running walk")
 		}},
 
+	// History is taken before anything else of a delivery off the trunk: a merge long past, heard only from
+	// the forge's word, is neither checked, held nor walked (novox/hq issue 313).
+	{From: []State{Proposed, Checked, Ready, Rejected}, Event: EvHistory, To: Retired,
+		Guard: "history: made from the forge's word for a merge on the trunk heard " + historyLate.String() + " or more " +
+			"after it happened — replayed when this owner first held the seat — and no walk of it open",
+		Holds: func(d *Delivery, f Facts) error {
+			if d.History == "" {
+				return errors.New("nothing says it is history")
+			}
+			return isHistory(d)
+		}},
+	{From: []State{Held}, Event: EvRetire, To: Retired, Act: true,
+		Guard: "a person's word, with why, for history only: made from the forge's word for a merge on the trunk heard " +
+			historyLate.String() + " or more after it happened, and no walk of it open — never released or delivered",
+		Holds: func(d *Delivery, f Facts) error {
+			if f.Why == "" || f.By == "" {
+				return errors.New("a retire says who and why")
+			}
+			return isHistory(d)
+		}},
+
 	{From: []State{Proposed, Checked, Ready, Rejected}, Event: EvNewHead, To: Superseded,
 		Guard: "a newer head of the same pull request",
 		Holds: func(d *Delivery, f Facts) error { return unless(d.NewerHead != "", "no newer head") }},
@@ -213,6 +240,28 @@ var Table = []Row{
 		Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "" && f.By != "", "a stop says who and why") }},
 }
 
+// historyLate is how long after its merge a merge heard only from the forge's word is history: the forge
+// announces a merge within seconds, so one heard this late was replayed from the bus's history.
+const historyLate = 15 * time.Minute
+
+// isHistory is whether a delivery is history, answering why not: made from the forge's word at its merge, on
+// the trunk, its merge heard historyLate or more after it happened, and no walk of it open or waiting — a
+// walk the controller keeps for it is a delivery, never history.
+func isHistory(d *Delivery) error {
+	switch {
+	case d.MergedAs == "":
+		return errors.New("it is not on the trunk")
+	case !d.FromForgesWord():
+		return errors.New("this owner heard its head: it is not history made from the forge's word")
+	case d.MergedAt.IsZero() || d.Created.Sub(d.MergedAt) < historyLate:
+		return fmt.Errorf("its merge was heard %s after it happened, within %s: a live merge, not history",
+			d.Created.Sub(d.MergedAt).Round(time.Second), historyLate)
+	case d.Walk != nil && (d.Walk.Waits || d.Walk.State == walkBuilding || d.Walk.State == walkRolling):
+		return fmt.Errorf("its walk %s is open: the controller delivers it", d.Walk.ID)
+	}
+	return nil
+}
+
 // rowFor is the row a state and event name; nil when the table holds none.
 func rowFor(from State, ev Event, act bool) *Row {
 	for i := range Table {

modules/mesh-delivery/cmd/mesh-delivery/table_test.go

@@ -31,6 +31,14 @@ func aDeliveryFor(r Row) (*Delivery, Facts) {
 		d.MergedAs, d.Walk = "aaaa1111", &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}
 	case EvMergedUnchecked:
 		d.MergedAs = "aaaa1111"
+	case EvHistory, EvRetire:
+		d.MergedAs, d.FromForge = "aaaa1111", true
+		d.MergedAt, d.Created = now.Add(-48*time.Hour), now
+		if r.Event == EvHistory {
+			d.History = "merged two days before this owner heard of it"
+		} else {
+			f.Why, f.By = "history adopted at the switch", "jochen"
+		}
 	case EvChecked, EvAccepted:
 		d.Check = &Verdict{Gate: "pass"}
 	case EvRefused:
@@ -95,7 +103,8 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing.
 
 func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) {
 	events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead,
-		EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop}
+		EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop,
+		EvHistory, EvRetire}
 	for _, from := range append([]State{None}, AllStates...) {
 		for _, ev := range events {
 			for _, act := range []bool{false, true} {

modules/mesh-delivery/cmd/mesh-delivery/verbs.go

@@ -471,3 +471,89 @@ func (h *Holder) Status(listening string) map[string]any {
 	}
 	return out
 }
+
+// RetiredAnswer is what `retire-history` answers.
+type RetiredAnswer struct {
+	Retired      int            `json:"retired"`
+	ByRepository map[string]int `json:"by-repository,omitempty"`
+	Dry          bool           `json:"dry,omitempty"`
+	// LeftHeld are the held deliveries the rule does not take, with why — a person's still.
+	LeftHeld int      `json:"left-held"`
+	Left     []string `json:"left,omitempty"`
+	Says     string   `json:"says"`
+}
+
+// RetireHistory is the `retire-history` verb (novox/hq issue 313): held deliveries that are history — made from
+// the forge's word for a merge on the trunk heard long after it happened, with no walk of theirs open — retired
+// on a person's word, never released or delivered. One by its id, refused when the rule does not take it; or
+// every held one, of one repository when named, each the rule does not take left held and named. Dry answers
+// what it would do and changes nothing.
+func (h *Holder) RetireHistory(id, repository, why, by string, dry bool) (*RetiredAnswer, error) {
+	if why == "" {
+		return nil, errors.New("a retire says why")
+	}
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	now := h.Now()
+	var held []*Delivery
+	if id != "" {
+		d := h.deliveries[id]
+		if d == nil {
+			return nil, fmt.Errorf("no delivery %q", id)
+		}
+		held = []*Delivery{d}
+	} else {
+		for _, d := range h.deliveries {
+			if d.State == Held && (repository == "" || strings.EqualFold(d.Repository, repository)) {
+				held = append(held, d)
+			}
+		}
+		sort.Slice(held, func(i, j int) bool { return held[i].ID < held[j].ID })
+	}
+	out := &RetiredAnswer{ByRepository: map[string]int{}, Dry: dry}
+	for _, d := range held {
+		// Tried on a copy first: a refused retire changes nothing, and a dry one nothing either.
+		probe := *d
+		probe.Transitions = append([]Transition(nil), d.Transitions...)
+		f := Facts{Now: now, By: by, Why: why}
+		if _, err := Apply(&probe, EvRetire, true, f); err != nil {
+			if id != "" {
+				return nil, err
+			}
+			out.LeftHeld++
+			if len(out.Left) < 25 {
+				out.Left = append(out.Left, err.Error())
+			}
+			continue
+		}
+		out.Retired++
+		out.ByRepository[d.Repository]++
+		if dry {
+			continue
+		}
+		t, _ := Apply(d, EvRetire, true, f)
+		d.History = "retired by " + by + ": " + why
+		d.HeldWhy = ""
+		// Its pull request is not told: the forge may no longer hold it, or holds another under its number.
+		d.Owed = withoutKinds(d.Owed, EffectView, EffectStatus)
+		h.moved(d, []Transition{t})
+	}
+	verb := "retired"
+	if dry {
+		verb = "would retire"
+	}
+	out.Says = fmt.Sprintf("%s %d held delivery(ies) as history; %d held left for a person", verb, out.Retired, out.LeftHeld)
+	h.Logf("[mesh-delivery] retire-history (%s): %s", why, out.Says)
+	return out, nil
+}
+
+// withoutKinds is an owed list without the effects of the kinds named.
+func withoutKinds(owed []Effect, kinds ...string) []Effect {
+	var out []Effect
+	for _, e := range owed {
+		if !containsString(kinds, e.Kind) {
+			out = append(out, e)
+		}
+	}
+	return out
+}

modules/mesh-delivery/module.json

@@ -17,7 +17,8 @@
         "recheck",
         "release",
         "stop",
-        "close"
+        "close",
+        "retire-history"
       ]
     }
   ],