diff --git a/modules/mesh-delivery/cmd/mesh-delivery/effects.go b/modules/mesh-delivery/cmd/mesh-delivery/effects.go index 3736e280..b2522fe8 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/effects.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/effects.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "regexp" "sort" "strings" "time" @@ -104,6 +105,8 @@ func statusOf(d *Delivery) (string, string) { return "warning", clip("superseded: "+why, 140) case Stopped: return "error", clip("stopped: "+why, 140) + case Retired: + return "success", "history: merged before this owner delivered anything, never walked" } return "pending", string(d.State) } @@ -186,6 +189,7 @@ func (h *Holder) Flush() { url string } h.mu.Lock() + now := h.Now() for id := range h.dirty { if d := h.deliveries[id]; d != nil { h.keep(d) @@ -208,6 +212,9 @@ func (h *Holder) Flush() { continue } for _, e := range d.Owed { + if e.Next != nil && now.Before(*e.Next) { + continue + } j := job{id: id, e: e, repo: [2]string{d.Owner(), d.Repo()}, num: d.Number, url: d.HTMLURL} if e.Kind == EffectView { j.body = ViewBody(d, h.groups[d.Group]) @@ -220,6 +227,9 @@ func (h *Holder) Flush() { continue } for _, e := range g.Owed { + if e.Next != nil && now.Before(*e.Next) { + continue + } jobs = append(jobs, job{id: id, group: true, e: e}) } } @@ -228,15 +238,26 @@ func (h *Holder) Flush() { for _, j := range jobs { err := h.do(j.e, j.repo, j.num, j.body, j.url, j.group) h.mu.Lock() + // Why it is given up, when it is: the forge's word that what it is for does not exist is final, and + // so is a failure tried retryLimit times, the last ones retryCap apart. + dropped := "" + switch { + case err != nil && forgeKinds[j.e.Kind] && doesNotExist(err): + dropped = "the forge answers that what it is for does not exist" + case err != nil && j.e.Tries+1 >= retryLimit: + dropped = fmt.Sprintf("tried %d times since %s", j.e.Tries+1, j.e.Since.UTC().Format(time.RFC3339)) + } owed := func(list []Effect) []Effect { for i, o := range list { if o.Kind == j.e.Kind && o.Commit == j.e.Commit && o.Context == j.e.Context && o.Line == j.e.Line && o.Since.Equal(j.e.Since) { - if err == nil { + if err == nil || dropped != "" { return append(list[:i:i], list[i+1:]...) } list[i].Tries++ list[i].Last = err.Error() + next := h.Now().Add(retryAfter(list[i].Tries)) + list[i].Next = &next return list } } @@ -251,13 +272,43 @@ func (h *Holder) Flush() { d.Owed = owed(d.Owed) h.keep(d) } + if dropped != "" { + h.refuse(fmt.Errorf("%s: its owed %s given up — %s: %v", j.id, j.e.Kind, dropped, err)) + } h.mu.Unlock() - if err != nil && j.e.Tries%10 == 0 { - h.Logf("[mesh-delivery] %s: %s not done yet (%v); tried again", j.id, j.e.Kind, err) + if err != nil && dropped == "" && j.e.Tries%10 == 0 { + h.Logf("[mesh-delivery] %s: %s not done yet (%v); tried again in %s", j.id, j.e.Kind, err, + retryAfter(j.e.Tries+1)) } } } +// An owed effect that fails is tried again later each time: retryFirst after the first failure, doubling up +// to retryCap, and given up after retryLimit tries — a little over two days — saying so (novox/hq issue 313). +const ( + retryFirst = 10 * time.Second + retryCap = time.Hour + retryLimit = 60 +) + +// retryAfter is how long after its tries-th failure an effect is tried again. +func retryAfter(tries int) time.Duration { + wait := retryFirst + for i := 1; i < tries && wait < retryCap; i++ { + wait *= 2 + } + return min(wait, retryCap) +} + +// forgeKinds are the effects done by the forge's holder. +var forgeKinds = map[string]bool{EffectNote: true, EffectView: true, EffectStatus: true} + +// notThere is the forge's own answer that what was asked of does not exist: a pull request whose number the +// forge no longer holds, a commit it does not know. Tried again it answers the same, for ever. +var notThere = regexp.MustCompile(`(?i)Gitea API \S+: (404\b|\d{3}\b.*\bdoes not exist\b)`) + +func doesNotExist(err error) bool { return err != nil && notThere.MatchString(err.Error()) } + // do is one owed effect. func (h *Holder) do(e Effect, repo [2]string, number int, body, url string, group bool) error { switch e.Kind { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index f27d4707..73506256 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -220,6 +220,9 @@ type fakeForge struct { views map[string]string statuses map[string]string down bool + // viewErr is what View answers instead, when set: a pull request the forge no longer holds. + viewErr error + viewCalls int // What Statuses reads: the pull requests by owner/repo#number, the base's required statuses, whether the // protection cannot be read, and what it was asked. pulls map[string]ForgePull @@ -251,6 +254,10 @@ func (f *fakeForge) Note(owner, repo, commit, line string) error { func (f *fakeForge) View(owner, repo string, number int, body string) error { f.mu.Lock() defer f.mu.Unlock() + f.viewCalls++ + if f.viewErr != nil { + return f.viewErr + } if f.down { return errors.New("the forge is away") } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/history_test.go b/modules/mesh-delivery/cmd/mesh-delivery/history_test.go new file mode 100644 index 00000000..382964e4 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/history_test.go @@ -0,0 +1,233 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" +) + +// History (novox/hq issue 313): merges long past, heard only from the forge's word when this owner first held +// the seat, reached the machines before it existed. They are retired — never held for a person, released or +// delivered — and what is owed for a pull request the forge no longer holds is given up, not tried for ever. + +// aPastMerge is the forge's `pull.merged` replayed from the bus's history: merged days before it is heard. +func aPastMerge(repo string, number int, head, merge string, at time.Time) PullEvent { + p := merged(repo, number, head, merge) + p.MergedAt = at.UTC().Format(time.RFC3339) + return p +} + +func TestAMergeLongPastHeardFromTheForgesWordIsRetiredAtAdoption(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullMerged(aPastMerge("novox/app", 274, head, merge, w.now.Add(-4*24*time.Hour))) + w.settleAll() + d := w.delivery(id) + if d.State != Retired || !strings.Contains(d.History, "history") { + t.Fatalf("a merge four days past heard from the forge's word is %s (%q)", d.State, d.History) + } + if got := statesOf([]State{d.Transitions[0].To, d.Transitions[1].To}); strings.Join(got, ",") != "proposed,retired" || + d.Transitions[1].Event != EvHistory { + t.Fatalf("transitions %+v", d.Transitions) + } + // Nothing is said on its pull request — the forge may hold another under its number — and nothing stalls. + if w.forge.viewCalls != 0 || len(w.forge.statuses) != 0 { + t.Fatalf("history was shown on a pull request: %d view(s), statuses %v", w.forge.viewCalls, w.forge.statuses) + } + if len(d.Owed) != 0 { + t.Fatalf("still owed: %+v", d.Owed) + } + if s := w.h.Stalled(); len(s) != 0 { + t.Fatalf("history stalls: %+v", s) + } + // Its record is still kept, on the commit's note. + if len(w.forge.notesOn(head)) == 0 { + t.Fatal("no note of it on its commit") + } + + // A merge heard from the forge's word within moments of happening is a live one: held for a person. + live := IDOf("novox/app", "bbbbbbbbbbbb2222") + w.h.PullMerged(aPastMerge("novox/app", 9, "bbbbbbbbbbbb2222", "cccccccccccc3333", w.now.Add(-time.Minute))) + if s := w.state(live); s != Held { + t.Fatalf("a live merge from the forge's word is %s", s) + } + // A merge long past whose walk the controller still holds open is delivered by it, never history. + walked := IDOf("novox/app", "dddddddddddd4444") + w.h.WalkMoved(aWalk("plan-9", "novox/app", "eeeeeeeeeeee5555", true)) + w.h.PullMerged(aPastMerge("novox/app", 10, "dddddddddddd4444", "eeeeeeeeeeee5555", w.now.Add(-48*time.Hour))) + if s := w.state(walked); s == Retired { + t.Fatal("a merge with an open walk was retired as history") + } + // A merge the forge announces under a number an older delivery already merged with is not that delivery. + w.h.PullMerged(aPastMerge("novox/app", 274, "", "ffffffffffff6666", w.now.Add(-time.Minute))) + if d := w.delivery(id); d.MergedAs != merge { + t.Fatalf("a reused number merged the old delivery again as %s", d.MergedAs) + } +} + +// oldHistory is a held delivery as the first adoption left it: made from the forge's word, held, owing a view +// on a pull request the forge no longer holds. +func oldHistory(id, repo string, number int, created, mergedAt time.Time) *Delivery { + commit := strings.SplitN(id, "@", 2)[1] + "0000" + return &Delivery{ID: id, Repository: repo, Commit: commit, Number: number, Base: "main", MergedAs: commit, + MergedAt: mergedAt, Created: created, Since: created, State: Held, + HeldWhy: "merged without a passing check: only a person decides that it goes on", + Transitions: []Transition{ + {At: created, To: Proposed, Event: EvAnnounced, Why: repo + "#274 merged; its head was not heard by this " + + "owner, so it is made from the forge's word"}, + {At: created, From: Proposed, To: Held, Event: EvMergedUnchecked, Why: "merged without a passing check"}}, + Owed: []Effect{{Kind: EffectView, Since: created, Tries: 11051, + Last: "Gitea API /repos/" + repo + "/issues/274/comments: 500 issue does not exist"}}} +} + +func TestRetireHistoryRetiresTheAdoptedHistoryInBulkAndLeavesEveryOtherHeldOne(t *testing.T) { + store := newMemStore() + start := time.Date(2026, 10, 6, 23, 5, 27, 0, time.UTC) + for _, d := range []*Delivery{ + oldHistory("novox/hq@018ee359ae52", "novox/hq", 274, start, start.Add(-4*24*time.Hour)), + oldHistory("novox/lab@a4f6bb8ce054", "novox/lab", 53, start, start.Add(-2*time.Hour)), + // Made from the forge's word, but heard a minute after its merge: a live merge, a person's. + oldHistory("novox/hq@7777777777aa", "novox/hq", 3, start, start.Add(-time.Minute)), + } { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + w := newWorldOver(t, store, newFakeController(), newFakeForge(), start.Add(36*time.Hour)) + // And one heard live: rejected, then merged — a person's to decide. + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "fail")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + if len(w.h.Stalled()) != 3 { // the three adopted at the switch; the one held just now is within its bound + t.Fatalf("stalled before: %+v", w.h.Stalled()) + } + + dry, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, true) + if err != nil || dry.Retired != 2 || dry.LeftHeld != 2 || len(dry.Left) != 2 { + t.Fatalf("a dry retire: %+v %v", dry, err) + } + if s := w.state("novox/hq@018ee359ae52"); s != Held { + t.Fatalf("a dry retire moved a delivery to %s", s) + } + if _, err := w.h.RetireHistory("", "", "", aPerson, false); err == nil { + t.Fatal("a retire with no why was taken") + } + + got, err := w.h.RetireHistory("", "", "history adopted at the switch", aPerson, false) + if err != nil || got.Retired != 2 || got.ByRepository["novox/hq"] != 1 || got.ByRepository["novox/lab"] != 1 { + t.Fatalf("retire: %+v %v", got, err) + } + for _, id := range []string{"novox/hq@018ee359ae52", "novox/lab@a4f6bb8ce054"} { + d := w.delivery(id) + if d.State != Retired || d.Transitions[len(d.Transitions)-1].By != aPerson { + t.Fatalf("%s is %s", id, d.State) + } + for _, e := range d.Owed { + if e.Kind == EffectView || e.Kind == EffectStatus { + t.Fatalf("%s still owes its pull request: %+v", id, e) + } + } + } + for _, id := range []string{"novox/hq@7777777777aa", IDOf("novox/app", head)} { + if s := w.state(id); s != Held { + t.Fatalf("%s, not history, is %s", id, s) + } + } + // Nothing was released, delivered or let go. + if len(w.ctl.delivers) != 0 || len(w.ctl.stops) != 0 { + t.Fatalf("a retire asked the controller: deliver %v stop %v", w.ctl.delivers, w.ctl.stops) + } + if s := w.h.Stalled(); len(s) != 1 || s[0].ID != "novox/hq@7777777777aa" { + t.Fatalf("stalled after: %+v", s) + } + // Retired is final: a second retire takes nothing, and nothing leaves it. + again, _ := w.h.RetireHistory("", "", "again", aPerson, false) + if again.Retired != 0 { + t.Fatalf("retired twice: %+v", again) + } + if _, err := w.h.Release("novox/hq@018ee359ae52", "go on", aPerson); err == nil { + t.Fatal("a retired delivery was released") + } +} + +func TestRetireHistoryIsRefusedForARealHeldDelivery(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "fail")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.later(48 * time.Hour) + _, err := w.h.RetireHistory(id, "", "looks old", aPerson, false) + var refused ErrRefused + if !errors.As(err, &refused) || !strings.Contains(err.Error(), "heard its head") { + t.Fatalf("a held delivery whose head was heard was retired: %v", err) + } + if s := w.state(id); s != Held { + t.Fatalf("a refused retire left it %s", s) + } + // A delivery that is not held is refused too, by the table. + other := IDOf("novox/app", "bbbbbbbbbbbb2222") + w.h.PullUpdated(pr("novox/app", 8, "bbbbbbbbbbbb2222", "feat/y", "src/b.go")) + if _, err := w.h.RetireHistory(other, "", "old", aPerson, false); err == nil || w.state(other) != Proposed { + t.Fatalf("a proposed delivery was retired: %v", err) + } + if _, err := w.h.RetireHistory("novox/app@nothing", "", "old", aPerson, false); err == nil { + t.Fatal("retired a delivery that does not exist") + } +} + +func TestOwedWorkTheForgeSaysCanNeverBeDoneIsDropped(t *testing.T) { + w := newWorld(t) + id := IDOf("novox/app", head) + w.forge.viewErr = errors.New(`Gitea API /repos/novox/app/issues/7/comments: 500 {"message":"issue does not ` + + `exist [id: 0, repo_id: 53, index: 7]"}`) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Flush() + if w.forge.viewCalls != 1 { + t.Fatalf("the view was asked %d time(s)", w.forge.viewCalls) + } + if d := w.delivery(id); len(d.Owed) != 0 { + t.Fatalf("an answer that its pull request does not exist left owed %+v", d.Owed) + } + w.settleAll() + if w.forge.viewCalls != 1 { + t.Fatalf("a dropped view was asked again: %d", w.forge.viewCalls) + } + if st := w.h.Status("listening"); !strings.Contains(strings.Join(st["refused"].([]string), "\n"), "given up") { + t.Fatalf("its drop was not said: %v", st["refused"]) + } + + // Any other failure is tried again later each time, and given up after retryLimit tries. + w.forge.viewErr = errors.New("the forge is away") + w.h.Checked(verdict("novox/app", 7, head, "pass")) + before := w.forge.viewCalls + w.h.Flush() + w.h.Flush() + w.h.Flush() + if w.forge.viewCalls != before+1 { + t.Fatalf("a failed view was tried %d time(s) at once", w.forge.viewCalls-before) + } + w.later(retryFirst + time.Second) + w.h.Flush() + if w.forge.viewCalls != before+2 { + t.Fatalf("a failed view was not tried again after its wait: %d", w.forge.viewCalls-before) + } + if retryAfter(1) != retryFirst || retryAfter(2) != 2*retryFirst || retryAfter(100) != retryCap { + t.Fatalf("the waits: %s %s %s", retryAfter(1), retryAfter(2), retryAfter(100)) + } + for range retryLimit { + w.later(retryCap) + w.h.Flush() + } + for _, e := range w.delivery(id).Owed { + if e.Kind == EffectView { + t.Fatalf("a view failing %d times is still owed: %+v", e.Tries, e) + } + } + if !doesNotExist(errors.New("Gitea API /repos/a/b/issues/1/comments: 404 Not Found")) || + doesNotExist(errors.New("tool gitea.gitea_delivery_view does not exist on this machine")) || + doesNotExist(errors.New("the forge is away")) { + t.Fatal("doesNotExist reads the forge's answer wrong") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index f2cc2360..84e96b68 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -138,7 +138,7 @@ func (h *Holder) moved(d *Delivery, ts []Transition) { d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: executedLine(d), Since: now}) } } - if len(ts) > 0 && d.Number > 0 { + if len(ts) > 0 && d.shownOnItsPullRequest() { d.owe(Effect{Kind: EffectView, Since: now}) state, desc := statusOf(d) d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) @@ -159,7 +159,8 @@ func (d *Delivery) owe(e Effect) { } func afterTheTrunk(s State) bool { - return s == Published || s == Delivering || s == Held || s == Delivered || s == Failed || s == Stopped || s == Superseded + return s == Published || s == Delivering || s == Held || s == Delivered || s == Failed || s == Stopped || + s == Superseded || s == Retired } func orNothing(s State) string { @@ -357,12 +358,15 @@ func (h *Holder) PullMerged(p PullEvent) { if d == nil { var newest time.Time for _, c := range h.deliveries { - if c.Repository == p.repository() && c.Number == p.Number && c.NewerHead == "" && c.Created.After(newest) { + // One already on the trunk is not merged again: a forge whose numbers began again reuses them. + if c.Repository == p.repository() && c.Number == p.Number && c.NewerHead == "" && c.MergedAs == "" && + c.Created.After(newest) { d, newest = c, c.Created } } } now := h.Now() + var ts []Transition if d == nil { head := p.HeadSHA if head == "" { @@ -371,7 +375,7 @@ func (h *Holder) PullMerged(p PullEvent) { d = &Delivery{ID: IDOf(p.repository(), head), Repository: p.repository(), Commit: head, Number: p.Number, Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, - After: AfterIn(p.Body), Created: now} + After: AfterIn(p.Body), Created: now, FromForge: true} // Its check, as the forge holds it on the head: the verdict the pull request merged with. if s := p.HeadChecks["mesh/merge-gate"]; s != "" { d.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it", @@ -383,7 +387,7 @@ func (h *Holder) PullMerged(p PullEvent) { h.refuse(err) return } - h.moved(d, []Transition{t}) + ts = append(ts, t) } d.MergedAs = p.MergeCommit if t, err := time.Parse(time.RFC3339, p.MergedAt); err == nil { @@ -398,7 +402,14 @@ func (h *Holder) PullMerged(p PullEvent) { delete(h.unmatched, id) } } - ts := settle(d, Facts{Now: now}) + // A merge long past heard only now, from the forge's word — replayed from the bus's history when this owner + // first held the seat — is history: it reached the machines before this owner was there to deliver it, and + // holding it for a person asks a person about the past (novox/hq issue 313). + if d.FromForge && d.History == "" && isHistory(d) == nil { + d.History = fmt.Sprintf("merged %s, %s before this owner heard of it from the forge's word: history, "+ + "never to be walked", d.MergedAt.UTC().Format(time.RFC3339), now.Sub(d.MergedAt).Round(time.Minute)) + } + ts = append(ts, settle(d, Facts{Now: now})...) if d.State == Held && d.HeldWhy == "" { d.HeldWhy = "merged without a passing check: only a person decides that it goes on" } @@ -1020,7 +1031,7 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { for i, m := range ordered { d := byID[m.ID] switch { - case d.State == Delivered: + case d.State == Delivered || d.State == Retired: continue case d.State == Failed || d.State == Stopped || d.State == Superseded: var asks []func() @@ -1082,7 +1093,7 @@ func GroupState(g *Group, members []*Delivery) (string, string) { return "failed", "a member failed" case count[Stopped] > 0 && g.Closed: return "stopped", "a member was stopped" - case n > 0 && count[Delivered] == n: + case n > 0 && count[Delivered]+count[Retired] == n: return "delivered", "every member is delivered" case g.Closed: return "delivering", fmt.Sprintf("%d of %d delivered", count[Delivered], n) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go index c028a5d1..dbc9ae79 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go @@ -256,6 +256,7 @@ func TestATransitionIsKeptBeforeItIsSaid(t *testing.T) { t.Fatal("noted while the forge was away") } w.forge.down = false + w.later(time.Hour) // each failure put its next try later: past every one of them w.settleAll() w.settleAll() if notes := w.forge.notesOn(head); len(notes) != 3 { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/main.go b/modules/mesh-delivery/cmd/mesh-delivery/main.go index c3c97b2a..20f3513b 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/main.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/main.go @@ -290,6 +290,25 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Close(strArg(a, "id"), strArg(a, "why")) }}, + {Name: seat + "retire-history", + Description: "Retire held deliveries that are history, on a person's word, with why: each made from the " + + "forge's word for a merge on the trunk heard 15 minutes or more after it happened — replayed when this " + + "owner first held the seat — with no walk of it open. Never released, delivered or put back: retired, " + + "final. One by id, refused when the rule does not take it; or every held one (of one repository when " + + "named), answering how many it retired and naming those it left held. dry \"true\" answers what it " + + "would do and changes nothing.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "why": str("why, kept with every transition"), "id": str("one delivery's id; every held one when empty"), + "repository": str("owner/repository: only its held deliveries"), + "dry": map[string]any{"type": "string", "enum": []string{"true", "false"}}}, + "required": []string{"why"}}, + Run: func(a map[string]any) (any, error) { + if err := need(a, "why"); err != nil { + return nil, err + } + return h.RetireHistory(strArg(a, "id"), strArg(a, "repository"), strArg(a, "why"), aPerson, + boolArg(a, "dry")) + }}, {Name: "delivery_status", Description: "Whether this owner works: how many deliveries in each state, the groups, what is owed to " + "the forge and the bus and not yet done, walks with no delivery yet, whether its events arrive, when " + diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 4239c96e..c778cbed 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -57,6 +57,10 @@ type Delivery struct { MergedAs string `json:"merged_as,omitempty"` MergedAt time.Time `json:"merged_at"` HeldWhy string `json:"held_why,omitempty"` + // FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner. + // History is why it is history, once it is (novox/hq issue 313). + FromForge bool `json:"from_forge,omitempty"` + History string `json:"history,omitempty"` // Walk is the controller's walk of its trunk commit, as last read; Steps the machines it reached. Walk *WalkSeen `json:"walk,omitempty"` Steps []Step `json:"steps,omitempty"` @@ -167,6 +171,8 @@ type Effect struct { Tries int `json:"tries,omitempty"` Last string `json:"last,omitempty"` Since time.Time `json:"since"` + // Next is when a failed effect is tried again: later after every failure, up to retryCap. + Next *time.Time `json:"next,omitempty"` } // The kinds of effect. @@ -177,6 +183,22 @@ const ( EffectStatus = "status" ) +// FromForgesWord is whether the delivery was made from the forge's word at its merge: said by FromForge, or — +// for one made before FromForge was kept — by its first transition. +func (d *Delivery) FromForgesWord() bool { + if d.FromForge { + return true + } + return len(d.Transitions) > 0 && d.Transitions[0].Event == EvAnnounced && + strings.Contains(d.Transitions[0].Why, "so it is made from the forge's word") +} + +// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose +// pull request the forge may no longer hold — or holds under its number another one. +func (d *Delivery) shownOnItsPullRequest() bool { + return d.Number > 0 && d.History == "" && d.State != Retired +} + // Owner and Repo split the repository. func (d *Delivery) Owner() string { o, _, _ := strings.Cut(d.Repository, "/"); return o } func (d *Delivery) Repo() string { _, r, _ := strings.Cut(d.Repository, "/"); return r } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index a4110e70..d6e1d7dd 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -30,15 +30,19 @@ const ( Failed State = "failed" Superseded State = "superseded" Stopped State = "stopped" + // Retired is history: a merge this owner heard only from the forge's word, long after it happened — + // replayed from the bus when it first held the seat. It reached the machines before this owner existed, + // and nothing is delivered, released or put back for it (novox/hq issue 313). + Retired State = "retired" ) // AllStates are every state a delivery can be in, for the table's own test and its verb. var AllStates = []State{Proposed, Checked, Ready, Rejected, Published, Delivering, Held, Delivered, Failed, - Superseded, Stopped} + Superseded, Stopped, Retired} // Final is whether nothing follows a state. func (s State) Final() bool { - return s == Delivered || s == Failed || s == Superseded || s == Stopped + return s == Delivered || s == Failed || s == Superseded || s == Stopped || s == Retired } // Event is what moves a delivery. @@ -66,6 +70,8 @@ const ( EvFailed Event = "failed" // its walk failed: a gate, a build, a machine EvSuperseded Event = "superseded" // a newer delivery to the same trunk took over its walk EvStop Event = "stop" // a person stopped it, or its group did + EvHistory Event = "history" // made from the forge's word for a merge long past: history + EvRetire Event = "retire" // a person's word that a held delivery is history, never to be walked ) // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. @@ -114,6 +120,27 @@ var Table = []Row{ return unless(d.Walk != nil && d.Walk.Started(), "no running walk") }}, + // History is taken before anything else of a delivery off the trunk: a merge long past, heard only from + // the forge's word, is neither checked, held nor walked (novox/hq issue 313). + {From: []State{Proposed, Checked, Ready, Rejected}, Event: EvHistory, To: Retired, + Guard: "history: made from the forge's word for a merge on the trunk heard " + historyLate.String() + " or more " + + "after it happened — replayed when this owner first held the seat — and no walk of it open", + Holds: func(d *Delivery, f Facts) error { + if d.History == "" { + return errors.New("nothing says it is history") + } + return isHistory(d) + }}, + {From: []State{Held}, Event: EvRetire, To: Retired, Act: true, + Guard: "a person's word, with why, for history only: made from the forge's word for a merge on the trunk heard " + + historyLate.String() + " or more after it happened, and no walk of it open — never released or delivered", + Holds: func(d *Delivery, f Facts) error { + if f.Why == "" || f.By == "" { + return errors.New("a retire says who and why") + } + return isHistory(d) + }}, + {From: []State{Proposed, Checked, Ready, Rejected}, Event: EvNewHead, To: Superseded, Guard: "a newer head of the same pull request", Holds: func(d *Delivery, f Facts) error { return unless(d.NewerHead != "", "no newer head") }}, @@ -213,6 +240,28 @@ var Table = []Row{ Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "" && f.By != "", "a stop says who and why") }}, } +// historyLate is how long after its merge a merge heard only from the forge's word is history: the forge +// announces a merge within seconds, so one heard this late was replayed from the bus's history. +const historyLate = 15 * time.Minute + +// isHistory is whether a delivery is history, answering why not: made from the forge's word at its merge, on +// the trunk, its merge heard historyLate or more after it happened, and no walk of it open or waiting — a +// walk the controller keeps for it is a delivery, never history. +func isHistory(d *Delivery) error { + switch { + case d.MergedAs == "": + return errors.New("it is not on the trunk") + case !d.FromForgesWord(): + return errors.New("this owner heard its head: it is not history made from the forge's word") + case d.MergedAt.IsZero() || d.Created.Sub(d.MergedAt) < historyLate: + return fmt.Errorf("its merge was heard %s after it happened, within %s: a live merge, not history", + d.Created.Sub(d.MergedAt).Round(time.Second), historyLate) + case d.Walk != nil && (d.Walk.Waits || d.Walk.State == walkBuilding || d.Walk.State == walkRolling): + return fmt.Errorf("its walk %s is open: the controller delivers it", d.Walk.ID) + } + return nil +} + // rowFor is the row a state and event name; nil when the table holds none. func rowFor(from State, ev Event, act bool) *Row { for i := range Table { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index 99c30810..3ea121c3 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -31,6 +31,14 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { d.MergedAs, d.Walk = "aaaa1111", &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} case EvMergedUnchecked: d.MergedAs = "aaaa1111" + case EvHistory, EvRetire: + d.MergedAs, d.FromForge = "aaaa1111", true + d.MergedAt, d.Created = now.Add(-48*time.Hour), now + if r.Event == EvHistory { + d.History = "merged two days before this owner heard of it" + } else { + f.Why, f.By = "history adopted at the switch", "jochen" + } case EvChecked, EvAccepted: d.Check = &Verdict{Gate: "pass"} case EvRefused: @@ -95,7 +103,8 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead, - EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop} + EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop, + EvHistory, EvRetire} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index 72f627d9..3edab64d 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -471,3 +471,89 @@ func (h *Holder) Status(listening string) map[string]any { } return out } + +// RetiredAnswer is what `retire-history` answers. +type RetiredAnswer struct { + Retired int `json:"retired"` + ByRepository map[string]int `json:"by-repository,omitempty"` + Dry bool `json:"dry,omitempty"` + // LeftHeld are the held deliveries the rule does not take, with why — a person's still. + LeftHeld int `json:"left-held"` + Left []string `json:"left,omitempty"` + Says string `json:"says"` +} + +// RetireHistory is the `retire-history` verb (novox/hq issue 313): held deliveries that are history — made from +// the forge's word for a merge on the trunk heard long after it happened, with no walk of theirs open — retired +// on a person's word, never released or delivered. One by its id, refused when the rule does not take it; or +// every held one, of one repository when named, each the rule does not take left held and named. Dry answers +// what it would do and changes nothing. +func (h *Holder) RetireHistory(id, repository, why, by string, dry bool) (*RetiredAnswer, error) { + if why == "" { + return nil, errors.New("a retire says why") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + var held []*Delivery + if id != "" { + d := h.deliveries[id] + if d == nil { + return nil, fmt.Errorf("no delivery %q", id) + } + held = []*Delivery{d} + } else { + for _, d := range h.deliveries { + if d.State == Held && (repository == "" || strings.EqualFold(d.Repository, repository)) { + held = append(held, d) + } + } + sort.Slice(held, func(i, j int) bool { return held[i].ID < held[j].ID }) + } + out := &RetiredAnswer{ByRepository: map[string]int{}, Dry: dry} + for _, d := range held { + // Tried on a copy first: a refused retire changes nothing, and a dry one nothing either. + probe := *d + probe.Transitions = append([]Transition(nil), d.Transitions...) + f := Facts{Now: now, By: by, Why: why} + if _, err := Apply(&probe, EvRetire, true, f); err != nil { + if id != "" { + return nil, err + } + out.LeftHeld++ + if len(out.Left) < 25 { + out.Left = append(out.Left, err.Error()) + } + continue + } + out.Retired++ + out.ByRepository[d.Repository]++ + if dry { + continue + } + t, _ := Apply(d, EvRetire, true, f) + d.History = "retired by " + by + ": " + why + d.HeldWhy = "" + // Its pull request is not told: the forge may no longer hold it, or holds another under its number. + d.Owed = withoutKinds(d.Owed, EffectView, EffectStatus) + h.moved(d, []Transition{t}) + } + verb := "retired" + if dry { + verb = "would retire" + } + out.Says = fmt.Sprintf("%s %d held delivery(ies) as history; %d held left for a person", verb, out.Retired, out.LeftHeld) + h.Logf("[mesh-delivery] retire-history (%s): %s", why, out.Says) + return out, nil +} + +// withoutKinds is an owed list without the effects of the kinds named. +func withoutKinds(owed []Effect, kinds ...string) []Effect { + var out []Effect + for _, e := range owed { + if !containsString(kinds, e.Kind) { + out = append(out, e) + } + } + return out +} diff --git a/modules/mesh-delivery/module.json b/modules/mesh-delivery/module.json index b247fa58..570e766f 100644 --- a/modules/mesh-delivery/module.json +++ b/modules/mesh-delivery/module.json @@ -17,7 +17,8 @@ "recheck", "release", "stop", - "close" + "close", + "retire-history" ] } ],