Give a rollout the word of a stopped delivery that waits for it (hq issue 753) #312

Merged
mesh-admin merged 1 commits from fix/753-a-rollout-is-never-left-without-a-word into main 2026-10-14 22:35:39 +00:00
4 changed files with 501 additions and 2 deletions
@@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool {
return false
}
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one.
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
// past the grace without its word it is listed by `stalled`, saying why it was not given.
func (h *Holder) heldWalk(w Walk) bool {
for _, d := range h.deliveries {
if d.State.Final() {
@@ -292,12 +294,25 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine {
continue
}
w := s.walk
if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 {
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
if why := h.lapsedSaid(w); why != "" {
says += ": " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
continue
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
out[len(out)-1].Says += h.endedSaid(w)
if hold := h.coverHeld(id); hold != "" {
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
}
@@ -347,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) {
}
case !w.Open() && now.Sub(s.at) > keptWalkFor:
delete(h.walks, id)
delete(h.deliverErr, id)
}
}
}
@@ -359,6 +375,168 @@ type turnWant struct {
whys []string
}
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
func (h *Holder) endedSaid(w Walk) string {
var ended []string
for _, d := range h.deliveries {
if !d.State.Final() {
continue
}
_, answers := walkAnswers(w, d)
if answers || d.Walk != nil && d.Walk.ID == w.ID {
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
}
if len(ended) == 0 {
return ""
}
sort.Strings(ended)
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
}
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
// itself, or by a person, gets nothing from it. Under the lock.
//
// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached
// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in
// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives
// nothing; lapsedBehind names that one.
func (h *Holder) lapsedOn(w Walk) []string {
out, _ := h.lapsedOf(w)
return out
}
// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that
// lapsed whose merge the walk answers. Under the lock.
func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) {
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
return nil, nil
}
var out []string
behind := map[string]string{}
for _, d := range h.deliveries {
if !h.lapsedAnswered(w, d) {
continue
}
if before := h.lapsedBehind(w, d); before != "" {
behind[d.ID] = before
continue
}
out = append(out, d.ID)
}
sort.Strings(out)
return out, behind
}
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
if d.MergedAs == "" || !h.stopLapsed(d) {
return false
}
if _, ok := walkAnswers(w, d); !ok {
return false
}
_, left := w.leftOut(d.Repository, d.MergedAs)
return !left
}
// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this
// walk does not deliver with it, with its state; empty for none.
func (h *Holder) lapsedBehind(w Walk, d *Delivery) string {
g := h.groups[d.Group]
if d.Group == "" || g == nil {
return ""
}
members := h.membersOf(g)
byID := map[string]*Delivery{}
for _, m := range members {
byID[m.ID] = m
}
for _, m := range h.inOrder(g, members) {
if m.ID == d.ID {
return ""
}
o := byID[m.ID]
if o == nil || memberReached(o) || h.lapsedAnswered(w, o) {
continue
}
return fmt.Sprintf("%s (%s)", o.ID, o.State)
}
return ""
}
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock.
func (h *Holder) lapsedSaid(w Walk) string {
var parts []string
_, behind := h.lapsedOf(w)
ids := make([]string, 0, len(behind))
for id := range behind {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id]))
}
if why := h.waitsWhy[w.ID]; why != "" {
parts = append(parts, "not let go yet: "+why)
}
if err := h.deliverErr[w.ID]; err != "" {
parts = append(parts, "the controller refused its word: "+err)
}
lapsed, _ := h.lapsedOf(w)
for _, id := range lapsed {
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
id, d.LetGo.Local().Format("15:04:05")))
}
}
return strings.Join(parts, "; ")
}
// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered.
func (h *Holder) lapsedOwed(w Walk) bool {
lapsed, behind := h.lapsedOf(w)
return len(lapsed) > 0 || len(behind) > 0
}
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
func (h *Holder) lapsedTurns() {
for id, s := range h.walks {
for _, did := range h.lapsedOn(s.walk) {
d := h.deliveries[did]
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
continue
}
if h.turning == nil {
h.turning = map[string]*turnWant{}
}
want := h.turning[id]
if want == nil {
want = &turnWant{}
h.turning[id] = want
}
if !containsString(want.ids, d.ID) {
want.ids = append(want.ids, d.ID)
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
}
}
}
}
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
// Under the lock.
@@ -409,9 +587,14 @@ func (h *Holder) words() []func() {
defer h.mu.Unlock()
delete(h.asking, walk)
if err != nil {
if h.deliverErr == nil {
h.deliverErr = map[string]string{}
}
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
return
}
delete(h.deliverErr, walk)
for _, id := range ids {
d := h.deliveries[id]
if d == nil {
@@ -492,6 +675,21 @@ func (h *Holder) WalksWaiting() []StalledLine {
case h.waitsWhy[id] != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
case !live && h.lapsedOwed(w):
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
// unheld, while the controller has not taken it, with what holds it.
lapsed, _ := h.lapsedOf(w)
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
"delivered since (novox/hq issue 753)"
if len(lapsed) > 0 {
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
"whose merge it answers (novox/hq issue 753)"
}
if why := h.lapsedSaid(w); why != "" {
says += "; " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: says})
case !live:
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
"`mesh-delivery.show` for the deliveries of " + w.Repository
@@ -502,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine {
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says})
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
}
}
sort.Slice(out, func(i, j int) bool {
@@ -106,6 +106,8 @@ type fakeController struct {
moves map[string][]string
plans int
down bool
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
refuseDeliver bool
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
onCheck func()
// planned is what Plan answers instead of "builds app", when set.
@@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error {
if c.down {
return errors.New("down")
}
if c.refuseDeliver {
return errors.New("the controller refuses the word")
}
w, known := c.walks[walk]
if !known || w.Delivery == nil || w.Delivery.Go != nil {
return fmt.Errorf("%s waits for nobody", walk)
@@ -60,6 +60,8 @@ type Holder struct {
waitsWhy map[string]string
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
asking map[string]bool
// deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753).
deliverErr map[string]string
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
leaving map[string][]string
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
@@ -1244,6 +1246,8 @@ func (h *Holder) plan() []func() {
}
h.turn(d, "its turn: a delivery in no group")
}
// A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753).
h.lapsedTurns()
// The word, once per walk, when every live delivery it carries is at its turn.
asks = append(asks, h.words()...)
@@ -0,0 +1,292 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
//
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
const (
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
plan753 = "plan-1792012015539062661"
)
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
t.Helper()
return night753Over(t, stop, nil, nil, more...)
}
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
// tick.
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
t.Helper()
at := at747
store := newMemStore()
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
if stop.By == byOwner {
m301.StoppedFor = id336
}
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
CoverTold: true,
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
inGroup, order := []*Delivery{m301, m336}, []string{id336}
for _, d := range more {
if d.Group == g747 {
inGroup, order = append(inGroup, d), append(order, d.ID)
}
}
order = append(order, id301)
var ids []string
for _, d := range inGroup {
ids = append(ids, d.ID)
}
g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering",
OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
{Repository: cat747, Commit: merge307}}}})
if len(merges) > 0 {
r := ctlr.walk(plan753)
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
ctlr.put(r)
}
if set != nil {
set(ctlr)
}
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
func groupStop753() Transition {
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
}
func saidOf(lines []StalledLine, walk string) (string, State) {
said, state := "", State("")
for _, l := range lines {
if l.Walk == walk {
said, state = said+l.Says, l.State
}
}
return said, state
}
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
w := night753(t, groupStop753())
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
}
d := w.delivery(id301)
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
t.Fatalf("the walk reads unheld: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk(plan753)
sent := w.now
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
t.Fatalf("#301 reads %s %q", state, desc)
}
}
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
// the walk's, never a way past a live delivery it carries.
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
at := at747
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
Waits: true, Created: at.Add(46 * time.Minute)}}
w := night753With(t, groupStop753(), held)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
}
}
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
// never let go for it.
func TestAPersonsStopGivesNoWord(t *testing.T) {
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
Why: "stopped by hand"})
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
}
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
}
}
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
// only that nothing holds it.
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
w := night753(t, groupStop753())
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
// gives it its word, and no live one holds it.
r := w.ctl.walk(plan753)
r.Delivery.Go = nil
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
w.h.WalkMoved(w.ctl.put(r))
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
}
w.settleAll()
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ = saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled` reads %q", said)
}
}
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
func m299() *Delivery {
at := at747
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
}
// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and
// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land
// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and
// is let go.
func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) {
w := night753With(t, groupStop753(), m299())
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers)
}
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") {
t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said)
}
w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil,
m299())
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers)
}
}
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
// the grace `stalled` lists the walk, with the refusal.
func TestARefusedWordIsListedByStalled(t *testing.T) {
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, state := saidOf(w.h.Stalled(), plan753)
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
t.Fatalf("`stalled` reads %s %q", state, said)
}
// Once the controller takes it, the walk is let go.
w.ctl.refuseDeliver = false
w.settleAll()
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
}
}
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
// grace `stalled` lists the walk, saying why.
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
at := at747
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ := saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
!strings.Contains(said, "whose delivery was stopped") {
t.Fatalf("`stalled` reads %q", said)
}
}