diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go index e3c65cb3..d62685cd 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/answers.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool { return false } -// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose +// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so +// past the grace without its word it is listed by `stalled`, saying why it was not given. func (h *Holder) heldWalk(w Walk) bool { for _, d := range h.deliveries { if d.State.Final() { @@ -292,12 +294,25 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine { continue } w := s.walk + if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 { + // Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why. + says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+ + "merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8)) + if why := h.lapsedSaid(w); why != "" { + says += ": " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: its word is given at its turn — the operator's past the bound", Says: says}) + continue + } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + out[len(out)-1].Says += h.endedSaid(w) if hold := h.coverHeld(id); hold != "" { out[len(out)-1].Says += "; it is held back meanwhile: " + hold } @@ -347,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) { } case !w.Open() && now.Sub(s.at) > keptWalkFor: delete(h.walks, id) + delete(h.deliverErr, id) } } } @@ -359,6 +375,168 @@ type turnWant struct { whys []string } +// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live +// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none. +func (h *Holder) endedSaid(w Walk) string { + var ended []string + for _, d := range h.deliveries { + if !d.State.Final() { + continue + } + _, answers := walkAnswers(w, d) + if answers || d.Walk != nil && d.Walk.ID == w.ID { + ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + } + if len(ended) == 0 { + return "" + } + sort.Strings(ended) + return "; every delivery it carries ended: " + strings.Join(ended, ", ") +} + +// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that +// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for +// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it, +// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so +// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for +// itself, or by a person, gets nothing from it. Under the lock. +// +// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached +// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in +// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives +// nothing; lapsedBehind names that one. +func (h *Holder) lapsedOn(w Walk) []string { + out, _ := h.lapsedOf(w) + return out +} + +// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that +// lapsed whose merge the walk answers. Under the lock. +func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) { + if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits { + return nil, nil + } + var out []string + behind := map[string]string{} + for _, d := range h.deliveries { + if !h.lapsedAnswered(w, d) { + continue + } + if before := h.lapsedBehind(w, d); before != "" { + behind[d.ID] = before + continue + } + out = append(out, d.ID) + } + sort.Strings(out) + return out, behind +} + +// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out. +func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool { + if d.MergedAs == "" || !h.stopLapsed(d) { + return false + } + if _, ok := walkAnswers(w, d); !ok { + return false + } + _, left := w.leftOut(d.Repository, d.MergedAs) + return !left +} + +// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this +// walk does not deliver with it, with its state; empty for none. +func (h *Holder) lapsedBehind(w Walk, d *Delivery) string { + g := h.groups[d.Group] + if d.Group == "" || g == nil { + return "" + } + members := h.membersOf(g) + byID := map[string]*Delivery{} + for _, m := range members { + byID[m.ID] = m + } + for _, m := range h.inOrder(g, members) { + if m.ID == d.ID { + return "" + } + o := byID[m.ID] + if o == nil || memberReached(o) || h.lapsedAnswered(w, o) { + continue + } + return fmt.Sprintf("%s (%s)", o.ID, o.State) + } + return "" +} + +// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet +// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock. +func (h *Holder) lapsedSaid(w Walk) string { + var parts []string + _, behind := h.lapsedOf(w) + ids := make([]string, 0, len(behind)) + for id := range behind { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id])) + } + if why := h.waitsWhy[w.ID]; why != "" { + parts = append(parts, "not let go yet: "+why) + } + if err := h.deliverErr[w.ID]; err != "" { + parts = append(parts, "the controller refused its word: "+err) + } + lapsed, _ := h.lapsedOf(w) + for _, id := range lapsed { + if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) { + parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it", + id, d.LetGo.Local().Format("15:04:05"))) + } + } + return strings.Join(parts, "; ") +} + +// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered. +func (h *Holder) lapsedOwed(w Walk) bool { + lapsed, behind := h.lapsedOf(w) + return len(lapsed) > 0 || len(behind) > 0 +} + +// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop +// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once, +// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words). +// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock. +func (h *Holder) lapsedTurns() { + for id, s := range h.walks { + for _, did := range h.lapsedOn(s.walk) { + d := h.deliveries[did] + if strings.Contains(d.LetGoWhy, lapsedWord(id)) { + continue + } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[id] + if want == nil { + want = &turnWant{} + h.turning[id] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+ + "was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+ + "walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository)) + } + } + } +} + +// lapsedWord is how the word a lapsed stop gives a walk names that walk. +func lapsedWord(walk string) string { return "the walk " + walk + " answers" } + // words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per // walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). // Under the lock. @@ -409,9 +587,14 @@ func (h *Holder) words() []func() { defer h.mu.Unlock() delete(h.asking, walk) if err != nil { + if h.deliverErr == nil { + h.deliverErr = map[string]string{} + } + h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753) h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) return } + delete(h.deliverErr, walk) for _, id := range ids { d := h.deliveries[id] if d == nil { @@ -492,6 +675,21 @@ func (h *Holder) WalksWaiting() []StalledLine { case h.waitsWhy[id] != "": out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) + case !live && h.lapsedOwed(w): + // Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never + // unheld, while the controller has not taken it, with what holds it. + lapsed, _ := h.lapsedOf(w) + says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " + + "delivered since (novox/hq issue 753)" + if len(lapsed) > 0 { + says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " + + "whose merge it answers (novox/hq issue 753)" + } + if why := h.lapsedSaid(w); why != "" { + says += "; " + why + } + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, + Bound: "0s", H2: "none: it goes at its turn", Says: says}) case !live: says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + "`mesh-delivery.show` for the deliveries of " + w.Repository @@ -502,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine { "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" } out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, - Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says}) + Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)}) } } sort.Slice(out, func(i, j int) bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index d6572f79..ded86f62 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -106,6 +106,8 @@ type fakeController struct { moves map[string][]string plans int down bool + // refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753). + refuseDeliver bool // onCheck runs as a check is asked, before it is answered: what the forge held at that moment. onCheck func() // planned is what Plan answers instead of "builds app", when set. @@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error { if c.down { return errors.New("down") } + if c.refuseDeliver { + return errors.New("the controller refuses the word") + } w, known := c.walks[walk] if !known || w.Delivery == nil || w.Delivery.Go != nil { return fmt.Errorf("%s waits for nobody", walk) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 383e2d67..a256007a 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -60,6 +60,8 @@ type Holder struct { waitsWhy map[string]string // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. asking map[string]bool + // deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753). + deliverErr map[string]string // leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299). leaving map[string][]string // leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped). @@ -1244,6 +1246,8 @@ func (h *Holder) plan() []func() { } h.turn(d, "its turn: a delivery in no group") } + // A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753). + h.lapsedTurns() // The word, once per walk, when every live delivery it carries is at its turn. asks = append(asks, h.words()...) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go new file mode 100644 index 00000000..8bb50c99 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue753_test.go @@ -0,0 +1,292 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at +// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its +// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery +// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later +// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's +// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word +// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries. +// +// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart. + +const ( + merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307" + plan753 = "plan-1792012015539062661" +) + +var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307) + +// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304 +// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting. +func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) } + +// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301. +func night753With(t *testing.T, stop Transition, more ...*Delivery) *world { + t.Helper() + return night753Over(t, stop, nil, nil, more...) +} + +// night753Over is night753With with more merges named by the walk and the controller set as given before the first +// tick. +func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world { + t.Helper() + at := at747 + store := newMemStore() + m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747, + State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at, + AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"), + Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true, + Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}} + m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747, + State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}} + if stop.By == byOwner { + m301.StoppedFor = id336 + } + m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events", + State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304, + MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"), + // Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747). + DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)", + CoverTold: true, + Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main", + TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)}, + Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop, + Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}} + m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check", + State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307, + MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")} + for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + inGroup, order := []*Delivery{m301, m336}, []string{id336} + for _, d := range more { + if d.Group == g747 { + inGroup, order = append(inGroup, d), append(order, d.ID) + } + } + order = append(order, id301) + var ids []string + for _, d := range inGroup { + ids = append(ids, d.ID) + } + g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering", + OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute), + Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}} + ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307}, + {Repository: cat747, Commit: merge307}}}}) + if len(merges) > 0 { + r := ctlr.walk(plan753) + r.Delivery.Merges = append(merges, r.Delivery.Merges...) + ctlr.put(r) + } + if set != nil { + set(ctlr) + } + ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt + w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since. +func groupStop753() Transition { + return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"} +} + +func saidOf(lines []StalledLine, walk string) (string, State) { + said, state := "", State("") + for _, l := range lines { + if l.Walk == walk { + said, state = said+l.Says, l.State + } + } + return said, state +} + +// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for +// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this +// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld. +func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) { + w := night753(t, groupStop753()) + if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 { + t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers) + } + d := w.delivery(id301) + if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") { + t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld { + t.Fatalf("the walk reads unheld: %q", said) + } + if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 { + t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops) + } + // Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747). + w.forge.contains[merge301+" in "+merge307] = true + r := w.ctl.walk(plan753) + sent := w.now + for _, m := range []string{"build-agent", "gitea", "route-proxy"} { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = walkDone + w.h.WalkMoved(w.ctl.put(r)) + w.later(time.Minute) + w.settleAll() + if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) { + t.Fatalf("#301 reads %s %q", state, desc) + } +} + +// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among +// the walk's, never a way past a live delivery it carries. +func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) { + at := at747 + held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308, + Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour), + MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main", + Waits: true, Created: at.Add(46 * time.Minute)}} + w := night753With(t, groupStop753(), held) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers) + } + if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held { + t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said) + } +} + +// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out, +// never let go for it. +func TestAPersonsStopGivesNoWord(t *testing.T) { + w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen", + Why: "stopped by hand"}) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers) + } + if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 { + t.Fatal("the walk carrying a person's stop was neither left without it nor stopped") + } +} + +// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it +// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never +// only that nothing holds it. +func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) { + w := night753(t, groupStop753()) + // The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed + // gives it its word, and no live one holds it. + r := w.ctl.walk(plan753) + r.Delivery.Go = nil + r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}} + w.h.WalkMoved(w.ctl.put(r)) + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state != Unheld || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307) + } + w.settleAll() + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ = saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") { + t.Fatalf("`stalled` reads %q", said) + } +} + +// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk. +func m299() *Delivery { + at := at747 + return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299, + Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute), + Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second), + Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"), + Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner, + Why: "its group " + g747 + " stopped: " + id336 + " failed"}}} +} + +// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and +// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land +// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and +// is let go. +func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) { + w := night753With(t, groupStop753(), m299()) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers) + } + said, state := saidOf(w.h.WalksWaiting(), plan753) + if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") { + t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said) + } + w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil, + m299()) + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers) + } +} + +// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past +// the grace `stalled` lists the walk, with the refusal. +func TestARefusedWordIsListedByStalled(t *testing.T) { + w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true }) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, state := saidOf(w.h.Stalled(), plan753) + if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") { + t.Fatalf("`stalled` reads %s %q", state, said) + } + // Once the controller takes it, the walk is let go. + w.ctl.refuseDeliver = false + w.settleAll() + if !containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers) + } +} + +// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person, +// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the +// grace `stalled` lists the walk, saying why. +func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) { + at := at747 + m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305, + Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"}, + Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready, + To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}} + w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}}, + func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305) + if containsString(w.ctl.delivers, plan753) { + t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers) + } + w.later(matchGrace + time.Minute) + w.settleAll() + said, _ := saidOf(w.h.Stalled(), plan753) + if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") || + !strings.Contains(said, "whose delivery was stopped") { + t.Fatalf("`stalled` reads %q", said) + } +}