From 51a85c599821fad38164bbf9f84c6365d67f6c4e Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 03:34:04 +0200 Subject: [PATCH 1/3] mesh-delivery: a member after a superseded one waits while a later merge can carry it Superseded is no failure: the later merge that took its walk over may still deliver its change, and a covered member counts as delivered (ADR 0270). So the members after it wait while such a merge is open within its own bound, take their turn once its delivered walk covers the member, and are stopped, saying why, only when none can carry it (novox/hq issue 445). --- .../mesh-delivery/cmd/mesh-delivery/holder.go | 103 ++++++++++- .../cmd/mesh-delivery/issue445_test.go | 161 ++++++++++++++++++ 2 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 360746eb..20c37bcb 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -1393,7 +1393,9 @@ func (h *Holder) inOrder(g *Group, members []*Delivery) []Member { // groupTurns walks a delivering group in its order: the next member is let go only when every member before // it is delivered; a member that failed or was stopped stops every member after it, naming it. What was // delivered before it stays, each judged at its own gate (ADR 0239 decision 4); the failed member's builds -// were put back at its gate by the controller. +// were put back at its gate by the controller. A superseded member counts as delivered once a later delivered +// walk carried it, holds the members after it while a later merge can still carry it, and stops them, saying +// why, only when none can (novox/hq issue 445). func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { ordered := h.inOrder(g, members) byID := map[string]*Delivery{} @@ -1409,18 +1411,33 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { sharing := "" for i, m := range ordered { d := byID[m.ID] - if d.State.Reached() { + // A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a + // covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq + // issues 442 and 445). + if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" { continue } if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { return asks } switch { - case d.State == Failed || d.State == Stopped || d.State == Superseded: - // A superseded member is an end here, and it stops the members after it even when a later delivered - // walk carried it. GroupState counts such a member as reached (novox/hq issue 442), so for a carried - // member the two do not agree: the members after it are stopped all the same, and the group then reads - // stopped for them. That gap is novox/hq issue 445, not closed here. + case d.State == Superseded: + // Superseded is no failure: the merge that took its walk over may still deliver its change. While a later + // merge into its trunk that can carry it is open, within that merge's own bound, the members after it + // wait, as for a member still delivering (ADR 0239 decision 4: the next starts when the one before is + // delivered). Only when none can carry it are they stopped, saying why (novox/hq issue 445). + awaited, why := h.carryAwaited(d, h.Now()) + if awaited != "" { + return asks + } + for _, later := range ordered[i+1:] { + if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, d.ID, + why)); a != nil { + asks = append(asks, a) + } + } + return asks + case d.State == Failed || d.State == Stopped: for _, later := range ordered[i+1:] { if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { asks = append(asks, a) @@ -1468,6 +1485,74 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { return asks } +// containsGrace is how long a later merge whose walk was delivered may still be awaited as a superseded member's +// carrier while the forge has not said whether its merge commit contains the member's: as long as a published +// delivery may wait for its turn (Bounds[Published]). A forge that cannot answer means "not yet covered" (ADR +// 0270), and without this bound the members after it would wait on that answer for ever (novox/hq issue 445). +const containsGrace = 30 * time.Minute + +// carryAwaited says which later merges into a superseded member's trunk may still carry every module it moves — +// the cover GroupState counts as reached (ADR 0270's covered test, novox/hq issue 442) — or, when none can, why. +// A later merge of the same repository into the same trunk on a passing check, whose plan moves the module and +// whose merge commit the forge has not said leaves this one out, can carry it: while it is merged and not yet +// ended, not held for a person, and within its own state's bound (ADR 0239 decision 9, ADR 0276 decision 9: every +// wait has a bound); once delivered, while the forge has not yet said whether it contains this merge, within +// containsGrace. So the wait is never longer than the carrier's own bound; a carrier superseded in turn hands the +// wait to the later merge that took it over, which is a carrier of its own (novox/hq issue 445). +func (h *Holder) carryAwaited(d *Delivery, now time.Time) (string, string) { + if d.MergedAs == "" || d.MergedAt.IsZero() || d.Plan == nil || len(d.Plan.Moved) == 0 { + return "", "nothing records which modules it moves, so no later merge can carry it" + } + var awaited []string + for _, module := range d.Plan.Moved { + found := false + var not []string + for _, c := range h.deliveries { + if !mayCarry(c, d, module) { + continue + } + contained, known := d.ContainedIn[c.MergedAs] + switch { + case known && !contained: + not = append(not, c.ID+" does not contain its merge") + case c.State == Delivered && carries(c, d, module) && known: + found = true // covered for this module; another module is what is still awaited + case c.State == Delivered && carries(c, d, module) && now.Sub(c.Since) <= containsGrace: + found = true + awaited = append(awaited, c.ID+" (delivered; whether it contains its merge is asked of the forge)") + case c.State == Delivered && carries(c, d, module): + not = append(not, fmt.Sprintf("the forge did not say within %s whether %s contains its merge", containsGrace, c.ID)) + case c.State.Final() || c.State == Held: + // ended without carrying it, or held for a person: no carrier to wait for + case pastBound(c, now): + b, _ := boundOf(c) + not = append(not, fmt.Sprintf("%s is past its bound (%s %s)", c.ID, c.State, b.For)) + default: + found = true + awaited = append(awaited, fmt.Sprintf("%s (%s)", c.ID, c.State)) + } + } + if !found { + why := fmt.Sprintf("no later merge into %s can carry %s", d.Base, module) + if len(not) > 0 { + sort.Strings(not) + why += ": " + strings.Join(not, "; ") + } + return "", why + } + } + sort.Strings(awaited) + return strings.Join(slices.Compact(awaited), ", "), "" +} + +// mayCarry is whether c is a later merge that could carry module for d, by what is known before its walk ends: of +// the same repository, merged later into the same trunk, on its own passing check, its plan moving the module. +func mayCarry(c, d *Delivery, module string) bool { + return c.ID != d.ID && strings.EqualFold(c.Repository, d.Repository) && strings.EqualFold(c.Base, d.Base) && + c.MergedAs != "" && !sameCommit(c.MergedAs, d.MergedAs) && c.MergedAt.After(d.MergedAt) && c.Check.Passes() && + c.Plan != nil && slices.Contains(c.Plan.Moved, module) +} + // stopInGroup stops a member after one that failed, and its walk if it has one open. func (h *Holder) stopInGroup(d *Delivery, why string) func() { if d == nil || d.State.Final() { @@ -1499,8 +1584,8 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() { // 0270's covered test); a closed group whose members all ended is then delivered, and otherwise stopped, naming // the member. A group with nothing in progress never reads delivering. // -// groupTurns does not yet agree for a carried member: it stops the members after any superseded one, carried or -// not, and a stopped member makes the group read stopped here. That gap is novox/hq issue 445. +// groupTurns agrees: the members after a carried member take their turn, and they wait while a later merge can still +// carry it (novox/hq issue 445). func GroupState(g *Group, members []*Delivery) (string, string) { count := map[State]int{} reached, ended := 0, 0 diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go new file mode 100644 index 00000000..3784fefe --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go @@ -0,0 +1,161 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 445: in a delivery group ordered [A, B], A's walk was superseded by the walk of a later merge C +// of A's repository that does not answer A's merge. groupTurns stopped B at once, although C's walk could still +// carry A's modules and A then counts as reached (ADR 0270 decision 2, issue 442). While a later walk that can +// carry A is open, B waits; once that walk is delivered and covers A, B takes its turn; when no later merge can +// carry A, or the one that could is past its own bound, B is stopped, saying why. +// +// The shape is the live group fix/assign-says-why-a-module-is-not-there as mesh-delivery showed it on +// 2026-10-11 (issue 442's test): the controller's member superseded first in the order, the lab's after it. +func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { + const ctl, lab, group = "novox/mesh-controller", "novox/mesh-lab", "fix/assign-says-why-a-module-is-not-there" + at := time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) + modules := []string{"build-agent", "mesh-controller", "route-proxy"} + const mergedA, mergedB, mergedC = "a5f53ef9ebdf3650", "77777777aaaaaaaa", "3f68a49500000000" + aID, bID, cID := IDOf(ctl, "8adb7f1a051941e9"), IDOf(lab, "33e63ee5b47a7552"), IDOf(ctl, "3f68a495c0c0c0c0") + + // A world in which A's walk was superseded by C's, C's walk runs, and B waits for its turn. + open := func(t *testing.T) *world { + a := &Delivery{ID: aID, Repository: ctl, Commit: "8adb7f1a051941e9", Number: 109, Base: "main", Branch: group, + Group: group, State: Superseded, Since: at.Add(4 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedA, + MergedAt: at, Plan: planMoving(ctl, modules...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-a5f53e", Commit: mergedA, State: walkSuperseded, Branch: "main", + Note: "superseded at tier 0 by plan-later (" + ctl + " at 3f68a495); planned there again"}, + Transitions: []Transition{{At: at.Add(4 * time.Minute), From: Delivering, To: Superseded, Event: EvSuperseded, + Why: "a newer delivery to the same trunk took over its walk"}}} + b := &Delivery{ID: bID, Repository: lab, Commit: "33e63ee5b47a7552", Number: 117, Base: "main", Branch: group, + Group: group, State: Published, Since: at.Add(3 * time.Minute), Created: at.Add(-2 * time.Hour), + MergedAs: mergedB, MergedAt: at.Add(2 * time.Minute), Plan: planMoving(lab, "mesh-lab"), + Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-lab", Commit: mergedB, State: walkBuilding, + Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at.Add(3 * time.Minute)}} + c := &Delivery{ID: cID, Repository: ctl, Commit: "3f68a495c0c0c0c0", Number: 110, Base: "main", + Branch: "fix/later", State: Delivering, Since: at.Add(5 * time.Minute), Created: at.Add(-time.Hour), + MergedAs: mergedC, MergedAt: at.Add(time.Minute), Plan: planMoving(ctl, modules...), + Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-later", Commit: mergedC, State: walkRolling, + WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at.Add(4 * time.Minute)}} + store := newMemStore() + for _, d := range []*Delivery{a, b, c} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: group, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, + Said: "delivering", OrderOf: map[string]string{}, + Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Hour), Verdict: "pass", Summary: "both compose", + Commits: commitsOf([]*Delivery{a, b})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + ctlr.put(Walk{ID: "plan-a5f53e", Repository: ctl, Branch: "main", Commit: mergedA, Created: at.Add(30 * time.Second), + State: walkSuperseded, Note: a.Walk.Note, Modules: map[string]*WalkModule{}}) + ctlr.put(Walk{ID: "plan-lab", Repository: lab, Branch: "main", Commit: mergedB, Created: at.Add(3 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"mesh-lab"}}, Modules: map[string]*WalkModule{"mesh-lab": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}) + went := at.Add(5 * time.Minute) + later := Walk{ID: "plan-later", Repository: ctl, Branch: "main", Commit: mergedC, Created: at.Add(4 * time.Minute), + State: walkRolling, Tiers: [][]string{modules}, Modules: map[string]*WalkModule{}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Go: &went, By: byOwner}} + for _, m := range modules { + later.Modules[m] = &WalkModule{} + } + ctlr.put(later) + w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(10*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w + } + // end ends C's walk in the state given, every module sent to every machine running it. + end := func(w *world, state string) { + walk := w.ctl.walk("plan-later") + sent := w.now + for _, m := range modules { + walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + walk.State = state + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + } + waits := func(t *testing.T, w *world) { + t.Helper() + if s := w.state(bID); s != Published || containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("B is %s, let go %v; want it published and waiting while C's walk can still carry A", s, w.ctl.delivers) + } + g := w.h.groups[group] + if state, why := GroupState(g, w.h.membersOf(g)); state != "delivering" { + t.Fatalf("the group reads %s (%s), want delivering", state, why) + } + } + stopped := func(t *testing.T, w *world, says string) { + t.Helper() + d := w.delivery(bID) + if d.State != Stopped || len(d.Transitions) == 0 { + t.Fatalf("B is %s, want stopped", d.State) + } + why := d.Transitions[len(d.Transitions)-1].Why + if !strings.Contains(why, aID) || !strings.Contains(why, says) { + t.Fatalf("B was stopped saying %q, want it to name %s and say %q", why, aID, says) + } + if containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("B's walk was let go %v", w.ctl.delivers) + } + } + + t.Run("a later walk that can carry A is open: B waits", func(t *testing.T) { + waits(t, open(t)) + }) + t.Run("that walk delivered and covers A: B takes its turn", func(t *testing.T) { + w := open(t) + w.forge.contains[mergedA+" in "+mergedC] = true + end(w, walkDone) + if s := w.state(cID); s != Delivered { + t.Fatalf("C is %s", s) + } + if d := w.delivery(aID); d.State != Superseded || d.CoveredBy == "" { + t.Fatalf("A is %s, covered by %q", d.State, d.CoveredBy) + } + if s := w.state(bID); s != Delivering || !containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("B is %s, let go %v; want its turn taken", s, w.ctl.delivers) + } + }) + t.Run("the forge says the later merge does not contain A: B is stopped", func(t *testing.T) { + w := open(t) + w.forge.contains[mergedA+" in "+mergedC] = false + end(w, walkDone) + stopped(t, w, "can carry") + }) + t.Run("the later walk failed: B is stopped", func(t *testing.T) { + w := open(t) + end(w, walkFailed) + if s := w.state(cID); s != Failed { + t.Fatalf("C is %s", s) + } + stopped(t, w, "can carry") + }) + t.Run("the later walk stays open past its own bound: B is stopped", func(t *testing.T) { + w := open(t) + w.later(Bounds[Delivering].For - 10*time.Minute) + w.settleAll() + waits(t, w) + w.later(10 * time.Minute) + w.settleAll() + stopped(t, w, "past its bound") + }) + t.Run("the forge never says whether the delivered walk contains A: B is stopped after the grace", func(t *testing.T) { + w := open(t) + end(w, walkDone) + waits(t, w) + w.later(containsGrace + time.Minute) + w.settleAll() + stopped(t, w, "did not say") + }) +} From f7ba413c1026a47a83877e409414e10c93437442 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 03:41:52 +0200 Subject: [PATCH 2/3] mesh-delivery: a member on its carrier's walk takes its turn, and a waiting member says for whom Under ADR 0276 the merge that carries a superseded member is often folded into the next member's own walk, which waits for that member's turn: waiting there waited for itself until the carrier's bound. The members on that walk now take their turn, as members sharing one walk do. A member that waits says which merge it waits for and until when, and `stalled` uses that bound; a carrier still in a batch is bounded by the window and one walk; a member every module of which was carried is reached, never stopped with no reason (novox/hq issue 445, review of #203). --- .../mesh-delivery/cmd/mesh-delivery/holder.go | 165 +++++-- .../cmd/mesh-delivery/issue445_test.go | 414 +++++++++++++----- .../mesh-delivery/cmd/mesh-delivery/model.go | 5 + .../mesh-delivery/cmd/mesh-delivery/table.go | 7 + 4 files changed, 444 insertions(+), 147 deletions(-) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 20c37bcb..b75f537e 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -1397,6 +1397,25 @@ func (h *Holder) inOrder(g *Group, members []*Delivery) []Member { // walk carried it, holds the members after it while a later merge can still carry it, and stops them, saying // why, only when none can (novox/hq issue 445). func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { + waits := map[string]carryWait{} + asks := h.turnsOf(g, members, waits) + // What each member's turn waits on a carrier for is kept on it, and cleared once it no longer does, so `show` + // and `stalled` name the carrier and until when (ADR 0276 decision 9, novox/hq issue 445). + for _, d := range members { + w := waits[d.ID] + if d.State != Published || !d.LetGo.IsZero() { + w = carryWait{} // only a member still waiting for its turn waits for a carrier + } + if d.TurnAwaits != w.said || !d.TurnAwaitsUntil.Equal(w.until) { + d.TurnAwaits, d.TurnAwaitsUntil = w.said, w.until + h.keep(d) + } + } + return asks +} + +// turnsOf is groupTurns' walk through the order; waits receives, by member, the carrier its turn waits for. +func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWait) []func() { ordered := h.inOrder(g, members) byID := map[string]*Delivery{} for _, d := range members { @@ -1426,15 +1445,32 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { // merge into its trunk that can carry it is open, within that merge's own bound, the members after it // wait, as for a member still delivering (ADR 0239 decision 4: the next starts when the one before is // delivered). Only when none can carry it are they stopped, saying why (novox/hq issue 445). - awaited, why := h.carryAwaited(d, h.Now()) - if awaited != "" { + w := h.carryAwaited(d, h.Now()) + if w.covered() { + continue // every module delivered by a later merge the forge says contains it: reached + } + if w.why != "" { + for _, later := range ordered[i+1:] { + if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, + d.ID, w.why)); a != nil { + asks = append(asks, a) + } + } return asks } + w.said = fmt.Sprintf("its turn in group %s waits for %s, superseded, to be carried by %s, until %s", g.ID, + d.ID, w.said, w.until.UTC().Format(time.RFC3339)) for _, later := range ordered[i+1:] { - if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, d.ID, - why)); a != nil { - asks = append(asks, a) - } + waits[later.ID] = w + } + // Under ADR 0276 a member is superseded when its waiting walk was folded into a successor that does not + // answer its merge, and the member after it was folded into the same successor: the carrier is then in + // the next member's own walk, which waits for that member's turn. Waiting there would wait for itself + // until the carrier's bound. That walk delivers both, in the group's order among its tiers (decision 5), + // so the members on it take their turn as members sharing one walk do; a member on another walk waits. + if shared := w.waitingWalk(); shared != "" && sharing == "" { + sharing = shared + continue } return asks case d.State == Failed || d.State == Stopped: @@ -1475,6 +1511,7 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { } if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil { asks, sharing = append(asks, a), d.Walk.ID + delete(waits, d.ID) continue } return asks @@ -1491,45 +1528,96 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { // 0270), and without this bound the members after it would wait on that answer for ever (novox/hq issue 445). const containsGrace = 30 * time.Minute -// carryAwaited says which later merges into a superseded member's trunk may still carry every module it moves — -// the cover GroupState counts as reached (ADR 0270's covered test, novox/hq issue 442) — or, when none can, why. -// A later merge of the same repository into the same trunk on a passing check, whose plan moves the module and -// whose merge commit the forge has not said leaves this one out, can carry it: while it is merged and not yet -// ended, not held for a person, and within its own state's bound (ADR 0239 decision 9, ADR 0276 decision 9: every -// wait has a bound); once delivered, while the forge has not yet said whether it contains this merge, within -// containsGrace. So the wait is never longer than the carrier's own bound; a carrier superseded in turn hands the -// wait to the later merge that took it over, which is a carrier of its own (novox/hq issue 445). -func (h *Holder) carryAwaited(d *Delivery, now time.Time) (string, string) { - if d.MergedAs == "" || d.MergedAt.IsZero() || d.Plan == nil || len(d.Plan.Moved) == 0 { - return "", "nothing records which modules it moves, so no later merge can carry it" +// readyCarrierBound is how long a merged carrier with no walk yet may be awaited from its merge. Its state, ready, +// has no bound of its own: it waits in a batch, which closes at the latest ten minutes after its first merge +// (ADR 0276 decision 1, `merge-window-at-most`), and then behind at most the one open walk, within that walk's +// bound (decisions 4 and 9). A merge no walk answers is held after `unwalked`, and a held one is no carrier. +const readyCarrierBound = 10*time.Minute + 2*time.Hour + +// carryWait is what a superseded member's group waits for: the later merges that may still carry it, said, and +// the moment the wait ends at the latest; or why none can carry it. Neither means every module it moves was +// delivered by a later merge the forge says contains it: covered. +type carryWait struct { + by []*Delivery + said string + until time.Time + why string +} + +func (w carryWait) covered() bool { return len(w.by) == 0 && w.why == "" } + +// waitingWalk is the walk of an awaited carrier that has not started: one waiting for this owner's word. +func (w carryWait) waitingWalk() string { + for _, c := range w.by { + if !c.State.Final() && c.Walk != nil && c.Walk.Waits && !c.Walk.Started() { + return c.Walk.ID + } } - var awaited []string + return "" +} + +// carryAwaited says which later merges into a superseded member's trunk may still carry every module it moves — +// the cover GroupState counts as reached (ADR 0270's covered test, novox/hq issue 442) — and until when, or, +// when none can, why. A later merge of the same repository into the same trunk on a passing check, whose plan +// moves the module and whose merge commit the forge has not said leaves this one out, can carry it while it is +// merged and not yet ended, not held for a person, and within its bound (ADR 0239 decision 9, ADR 0276 decision +// 9: every wait has a bound): its state's, or readyCarrierBound from its merge while no walk carries it yet; once +// delivered, while the forge has not yet said whether it contains this merge, within containsGrace. A carrier +// superseded in turn ends; the later merge that took it over is a carrier of its own (novox/hq issue 445). +func (h *Holder) carryAwaited(d *Delivery, now time.Time) carryWait { + if d.MergedAs == "" || d.MergedAt.IsZero() || d.Plan == nil || len(d.Plan.Moved) == 0 { + return carryWait{why: "nothing records which modules it moves, so no later merge can carry it"} + } + var w carryWait + seen := map[string]bool{} +modules: for _, module := range d.Plan.Moved { found := false + var latest time.Time // the last moment a carrier of this module may still be awaited var not []string for _, c := range h.deliveries { if !mayCarry(c, d, module) { continue } contained, known := d.ContainedIn[c.MergedAs] + var until time.Time switch { case known && !contained: not = append(not, c.ID+" does not contain its merge") + continue case c.State == Delivered && carries(c, d, module) && known: - found = true // covered for this module; another module is what is still awaited - case c.State == Delivered && carries(c, d, module) && now.Sub(c.Since) <= containsGrace: - found = true - awaited = append(awaited, c.ID+" (delivered; whether it contains its merge is asked of the forge)") + continue modules // delivered and contained: this module is carried case c.State == Delivered && carries(c, d, module): - not = append(not, fmt.Sprintf("the forge did not say within %s whether %s contains its merge", containsGrace, c.ID)) + until = c.Since.Add(containsGrace) + if now.After(until) { + not = append(not, fmt.Sprintf("the forge did not say within %s whether %s contains its merge", containsGrace, c.ID)) + continue + } case c.State.Final() || c.State == Held: - // ended without carrying it, or held for a person: no carrier to wait for - case pastBound(c, now): - b, _ := boundOf(c) - not = append(not, fmt.Sprintf("%s is past its bound (%s %s)", c.ID, c.State, b.For)) + continue // ended without carrying it, or held for a person: no carrier to wait for + case c.State == Ready: + until = c.MergedAt.Add(readyCarrierBound) + if now.After(until) { + not = append(not, fmt.Sprintf("%s is merged and no walk carried it within %s", c.ID, readyCarrierBound)) + continue + } default: - found = true - awaited = append(awaited, fmt.Sprintf("%s (%s)", c.ID, c.State)) + b, bounded := boundOf(c) + if pastBound(c, now) { + not = append(not, fmt.Sprintf("%s is past its bound (%s %s)", c.ID, c.State, b.For)) + continue + } + if until = now.Add(Bounds[Delivering].For); bounded && b.For > 0 { + until = boundSince(c).Add(b.For) + } + } + found = true + if until.After(latest) { + latest = until + } + if !seen[c.ID] { + seen[c.ID] = true + w.by = append(w.by, c) } } if !found { @@ -1538,11 +1626,24 @@ func (h *Holder) carryAwaited(d *Delivery, now time.Time) (string, string) { sort.Strings(not) why += ": " + strings.Join(not, "; ") } - return "", why + return carryWait{why: why} + } + // The wait ends when the first module runs out of carriers, whatever the others still have. + if w.until.IsZero() || latest.Before(w.until) { + w.until = latest } } - sort.Strings(awaited) - return strings.Join(slices.Compact(awaited), ", "), "" + sort.Slice(w.by, func(i, j int) bool { return w.by[i].ID < w.by[j].ID }) + var said []string + for _, c := range w.by { + s := fmt.Sprintf("%s (%s", c.ID, c.State) + if c.Walk != nil { + s += ", walk " + c.Walk.ID + } + said = append(said, s+")") + } + w.said = strings.Join(said, ", ") + return w } // mayCarry is whether c is a later merge that could carry module for d, by what is known before its walk ends: of diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go index 3784fefe..7215ca0f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go @@ -8,154 +8,338 @@ import ( // novox/hq issue 445: in a delivery group ordered [A, B], A's walk was superseded by the walk of a later merge C // of A's repository that does not answer A's merge. groupTurns stopped B at once, although C's walk could still -// carry A's modules and A then counts as reached (ADR 0270 decision 2, issue 442). While a later walk that can -// carry A is open, B waits; once that walk is delivered and covers A, B takes its turn; when no later merge can -// carry A, or the one that could is past its own bound, B is stopped, saying why. +// carry A's modules and A then counts as reached (ADR 0270 decision 2, issue 442). While a later merge that can +// carry A is open, B waits, saying for which merge and until when; once that merge's walk is delivered and covers +// A, B takes its turn; when C's walk is B's own (ADR 0276 folds both into one successor), B takes its turn on it; +// when no later merge can carry A, or the one that could is past its own bound, B is stopped, saying why. // // The shape is the live group fix/assign-says-why-a-module-is-not-there as mesh-delivery showed it on // 2026-10-11 (issue 442's test): the controller's member superseded first in the order, the lab's after it. -func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { - const ctl, lab, group = "novox/mesh-controller", "novox/mesh-lab", "fix/assign-says-why-a-module-is-not-there" - at := time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) - modules := []string{"build-agent", "mesh-controller", "route-proxy"} - const mergedA, mergedB, mergedC = "a5f53ef9ebdf3650", "77777777aaaaaaaa", "3f68a49500000000" - aID, bID, cID := IDOf(ctl, "8adb7f1a051941e9"), IDOf(lab, "33e63ee5b47a7552"), IDOf(ctl, "3f68a495c0c0c0c0") - // A world in which A's walk was superseded by C's, C's walk runs, and B waits for its turn. - open := func(t *testing.T) *world { - a := &Delivery{ID: aID, Repository: ctl, Commit: "8adb7f1a051941e9", Number: 109, Base: "main", Branch: group, - Group: group, State: Superseded, Since: at.Add(4 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedA, - MergedAt: at, Plan: planMoving(ctl, modules...), Check: &Verdict{Gate: "pass"}, - Walk: &WalkSeen{ID: "plan-a5f53e", Commit: mergedA, State: walkSuperseded, Branch: "main", - Note: "superseded at tier 0 by plan-later (" + ctl + " at 3f68a495); planned there again"}, - Transitions: []Transition{{At: at.Add(4 * time.Minute), From: Delivering, To: Superseded, Event: EvSuperseded, - Why: "a newer delivery to the same trunk took over its walk"}}} - b := &Delivery{ID: bID, Repository: lab, Commit: "33e63ee5b47a7552", Number: 117, Base: "main", Branch: group, - Group: group, State: Published, Since: at.Add(3 * time.Minute), Created: at.Add(-2 * time.Hour), - MergedAs: mergedB, MergedAt: at.Add(2 * time.Minute), Plan: planMoving(lab, "mesh-lab"), - Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-lab", Commit: mergedB, State: walkBuilding, - Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at.Add(3 * time.Minute)}} - c := &Delivery{ID: cID, Repository: ctl, Commit: "3f68a495c0c0c0c0", Number: 110, Base: "main", - Branch: "fix/later", State: Delivering, Since: at.Add(5 * time.Minute), Created: at.Add(-time.Hour), - MergedAs: mergedC, MergedAt: at.Add(time.Minute), Plan: planMoving(ctl, modules...), - Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-later", Commit: mergedC, State: walkRolling, - WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at.Add(4 * time.Minute)}} - store := newMemStore() - for _, d := range []*Delivery{a, b, c} { - if err := store.PutDelivery(d); err != nil { - t.Fatal(err) +const ( + g445 = "fix/assign-says-why-a-module-is-not-there" + ctl445, lab445 = "novox/mesh-controller", "novox/mesh-lab" + mergedA, mergedB, mergedC = "a5f53ef9ebdf3650", "77777777aaaaaaaa", "3f68a49500000000" + mergedD = "4d4d4d4d00000000" + headA, headB, headC, headD = "8adb7f1a051941e9", "33e63ee5b47a7552", "3f68a495c0c0c0c0", "4d4d4d4dc0c0c0c0" +) + +var ( + at445 = time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) + modules445 = []string{"build-agent", "mesh-controller", "route-proxy"} + aID, bID, cID, dID445 = IDOf(ctl445, headA), IDOf(lab445, headB), IDOf(ctl445, headC), IDOf(ctl445, headD) +) + +// shape445 is A superseded, B published on its walk, and C, the later merge of A's repository, as each case sets it. +type shape445 struct { + a, b, c *Delivery + extra []*Delivery + walks []Walk +} + +func newShape445() *shape445 { + at := at445 + s := &shape445{} + s.a = &Delivery{ID: aID, Repository: ctl445, Commit: headA, Number: 109, Base: "main", Branch: g445, Group: g445, + State: Superseded, Since: at.Add(4 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedA, MergedAt: at, + Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-a5f53e", Commit: mergedA, State: walkSuperseded, Branch: "main", + Note: "superseded at tier 0 by plan-later (" + ctl445 + " at 3f68a495); planned there again"}, + Transitions: []Transition{{At: at.Add(4 * time.Minute), From: Published, To: Superseded, Event: EvSuperseded, + Why: "a newer delivery to the same trunk took over its walk"}}} + s.b = &Delivery{ID: bID, Repository: lab445, Commit: headB, Number: 117, Base: "main", Branch: g445, Group: g445, + State: Published, Since: at.Add(3 * time.Minute), Created: at.Add(-2 * time.Hour), MergedAs: mergedB, + MergedAt: at.Add(2 * time.Minute), Plan: planMoving(lab445, "mesh-lab"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-lab", Commit: mergedB, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", + Branch: "main", Created: at.Add(3 * time.Minute)}} + s.c = &Delivery{ID: cID, Repository: ctl445, Commit: headC, Number: 110, Base: "main", Branch: "fix/later", + State: Delivering, Since: at.Add(5 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedC, + MergedAt: at.Add(time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-later", Commit: mergedC, State: walkRolling, WaitedFor: "mesh-delivery", GoBy: byOwner, + Branch: "main", Created: at.Add(4 * time.Minute)}} + went := at.Add(5 * time.Minute) + s.walks = []Walk{ + {ID: "plan-a5f53e", Repository: ctl445, Branch: "main", Commit: mergedA, Created: at.Add(30 * time.Second), + State: walkSuperseded, Note: s.a.Walk.Note, Modules: map[string]*WalkModule{}}, + {ID: "plan-lab", Repository: lab445, Branch: "main", Commit: mergedB, Created: at.Add(3 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"mesh-lab"}}, Modules: map[string]*WalkModule{"mesh-lab": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}, + ctlWalk("plan-later", mergedC, at.Add(4*time.Minute), walkRolling, &WalkDelivery{Awaits: "mesh-delivery", + Go: &went, By: byOwner}), + } + return s +} + +// ctlWalk is a walk of the controller's repository at a merge commit, moving A's modules. +func ctlWalk(id, commit string, created time.Time, state string, wd *WalkDelivery) Walk { + w := Walk{ID: id, Repository: ctl445, Branch: "main", Commit: commit, Created: created, State: state, + Tiers: [][]string{modules445}, Modules: map[string]*WalkModule{}, Delivery: wd} + for _, m := range modules445 { + w.Modules[m] = &WalkModule{} + } + return w +} + +// open is the world over the shape, read back and ticked ten minutes after A's merge. +func (s *shape445) open(t *testing.T) *world { + t.Helper() + store := newMemStore() + for _, d := range append([]*Delivery{s.a, s.b, s.c}, s.extra...) { + if d == nil { + continue + } + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", + OrderOf: map[string]string{}, Check: &GroupCheck{Asked: "check-1", At: at445.Add(-time.Hour), Verdict: "pass", + Summary: "both compose", Commits: commitsOf([]*Delivery{s.a, s.b})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + for _, w := range s.walks { + ctlr.put(w) + } + w := newWorldOver(t, store, ctlr, newFakeForge(), at445.Add(10*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// end ends a walk in the state given, every module sent to every machine running it. +func end445(w *world, walk, state string) { + r := w.ctl.walk(walk) + sent := w.now + for m := range r.Modules { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = state + w.h.WalkMoved(w.ctl.put(r)) + w.settleAll() +} + +func waits445(t *testing.T, w *world, on string) { + t.Helper() + d := w.delivery(bID) + if d.State != Published || containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("B is %s, let go %v; want it published and waiting while %s can still carry A", d.State, w.ctl.delivers, on) + } + if !strings.Contains(d.TurnAwaits, on) || !strings.Contains(d.TurnAwaits, aID) || d.TurnAwaitsUntil.IsZero() { + t.Fatalf("B says it waits for %q until %v; want it to name A and %s", d.TurnAwaits, d.TurnAwaitsUntil, on) + } + g := w.h.groups[g445] + if state, why := GroupState(g, w.h.membersOf(g)); state != "delivering" { + t.Fatalf("the group reads %s (%s), want delivering", state, why) + } +} + +func stopped445(t *testing.T, w *world, says string) { + t.Helper() + d := w.delivery(bID) + if d.State != Stopped || len(d.Transitions) == 0 { + t.Fatalf("B is %s, want stopped", d.State) + } + why := d.Transitions[len(d.Transitions)-1].Why + if !strings.Contains(why, aID) || !strings.Contains(why, says) { + t.Fatalf("B was stopped saying %q, want it to name %s and say %q", why, aID, says) + } + if containsString(w.ctl.delivers, "plan-lab") || d.TurnAwaits != "" { + t.Fatalf("B's walk was let go %v, or B still says it waits: %q", w.ctl.delivers, d.TurnAwaits) + } +} + +func letGo445(t *testing.T, w *world, walk string) { + t.Helper() + if d := w.delivery(bID); d.State != Delivering || !containsString(w.ctl.delivers, walk) || d.TurnAwaits != "" { + t.Fatalf("B is %s, let go %v, waiting for %q; want its turn taken on %s", d.State, w.ctl.delivers, d.TurnAwaits, walk) + } +} + +// C's walk runs on its own: B waits for it, says so, and goes on or stops by what comes of it. +func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { + t.Run("a later walk that can carry A is open: B waits, naming it", func(t *testing.T) { + w := newShape445().open(t) + waits445(t, w, cID) + // Past a published delivery's own 30 minutes, B is not stalled for "its turn or its word": its bound is + // C's, and what `stalled` would say names C. + w.later(Bounds[Published].For + 5*time.Minute) + w.settleAll() + waits445(t, w, cID) + for _, l := range w.h.Stalled() { + if l.ID == bID { + t.Fatalf("B is listed stalled while C's walk is within its bound: %+v", l) } } - g := &Group{ID: group, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, - Said: "delivering", OrderOf: map[string]string{}, - Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Hour), Verdict: "pass", Summary: "both compose", - Commits: commitsOf([]*Delivery{a, b})}} - if err := store.PutGroup(g); err != nil { - t.Fatal(err) + if b, _ := boundOf(w.delivery(bID)); !strings.Contains(b.Says, cID) || !strings.Contains(b.Says, "until") { + t.Fatalf("B's bound says %q", b.Says) } - ctlr := newFakeController() - ctlr.put(Walk{ID: "plan-a5f53e", Repository: ctl, Branch: "main", Commit: mergedA, Created: at.Add(30 * time.Second), - State: walkSuperseded, Note: a.Walk.Note, Modules: map[string]*WalkModule{}}) - ctlr.put(Walk{ID: "plan-lab", Repository: lab, Branch: "main", Commit: mergedB, Created: at.Add(3 * time.Minute), - State: walkBuilding, Tiers: [][]string{{"mesh-lab"}}, Modules: map[string]*WalkModule{"mesh-lab": {}}, - Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}) - went := at.Add(5 * time.Minute) - later := Walk{ID: "plan-later", Repository: ctl, Branch: "main", Commit: mergedC, Created: at.Add(4 * time.Minute), - State: walkRolling, Tiers: [][]string{modules}, Modules: map[string]*WalkModule{}, - Delivery: &WalkDelivery{Awaits: "mesh-delivery", Go: &went, By: byOwner}} - for _, m := range modules { - later.Modules[m] = &WalkModule{} - } - ctlr.put(later) - w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(10*time.Minute)) - if err := w.h.Reconcile(); err != nil { - t.Fatal(err) - } - w.settleAll() - return w - } - // end ends C's walk in the state given, every module sent to every machine running it. - end := func(w *world, state string) { - walk := w.ctl.walk("plan-later") - sent := w.now - for _, m := range modules { - walk.Modules[m] = &WalkModule{State: "built", SentAt: &sent} - } - walk.State = state - w.h.WalkMoved(w.ctl.put(walk)) - w.settleAll() - } - waits := func(t *testing.T, w *world) { - t.Helper() - if s := w.state(bID); s != Published || containsString(w.ctl.delivers, "plan-lab") { - t.Fatalf("B is %s, let go %v; want it published and waiting while C's walk can still carry A", s, w.ctl.delivers) - } - g := w.h.groups[group] - if state, why := GroupState(g, w.h.membersOf(g)); state != "delivering" { - t.Fatalf("the group reads %s (%s), want delivering", state, why) - } - } - stopped := func(t *testing.T, w *world, says string) { - t.Helper() - d := w.delivery(bID) - if d.State != Stopped || len(d.Transitions) == 0 { - t.Fatalf("B is %s, want stopped", d.State) - } - why := d.Transitions[len(d.Transitions)-1].Why - if !strings.Contains(why, aID) || !strings.Contains(why, says) { - t.Fatalf("B was stopped saying %q, want it to name %s and say %q", why, aID, says) - } - if containsString(w.ctl.delivers, "plan-lab") { - t.Fatalf("B's walk was let go %v", w.ctl.delivers) - } - } - - t.Run("a later walk that can carry A is open: B waits", func(t *testing.T) { - waits(t, open(t)) }) t.Run("that walk delivered and covers A: B takes its turn", func(t *testing.T) { - w := open(t) + w := newShape445().open(t) w.forge.contains[mergedA+" in "+mergedC] = true - end(w, walkDone) + end445(w, "plan-later", walkDone) if s := w.state(cID); s != Delivered { t.Fatalf("C is %s", s) } if d := w.delivery(aID); d.State != Superseded || d.CoveredBy == "" { t.Fatalf("A is %s, covered by %q", d.State, d.CoveredBy) } - if s := w.state(bID); s != Delivering || !containsString(w.ctl.delivers, "plan-lab") { - t.Fatalf("B is %s, let go %v; want its turn taken", s, w.ctl.delivers) - } + letGo445(t, w, "plan-lab") }) t.Run("the forge says the later merge does not contain A: B is stopped", func(t *testing.T) { - w := open(t) + w := newShape445().open(t) w.forge.contains[mergedA+" in "+mergedC] = false - end(w, walkDone) - stopped(t, w, "can carry") + end445(w, "plan-later", walkDone) + stopped445(t, w, "can carry") }) t.Run("the later walk failed: B is stopped", func(t *testing.T) { - w := open(t) - end(w, walkFailed) + w := newShape445().open(t) + end445(w, "plan-later", walkFailed) if s := w.state(cID); s != Failed { t.Fatalf("C is %s", s) } - stopped(t, w, "can carry") + stopped445(t, w, "can carry") }) t.Run("the later walk stays open past its own bound: B is stopped", func(t *testing.T) { - w := open(t) + w := newShape445().open(t) w.later(Bounds[Delivering].For - 10*time.Minute) w.settleAll() - waits(t, w) + waits445(t, w, cID) w.later(10 * time.Minute) w.settleAll() - stopped(t, w, "past its bound") + stopped445(t, w, "past its bound") }) t.Run("the forge never says whether the delivered walk contains A: B is stopped after the grace", func(t *testing.T) { - w := open(t) - end(w, walkDone) - waits(t, w) + w := newShape445().open(t) + end445(w, "plan-later", walkDone) + waits445(t, w, cID) w.later(containsGrace + time.Minute) w.settleAll() - stopped(t, w, "did not say") + stopped445(t, w, "did not say") }) } + +// The review's shape of #203 (ADR 0276): A's waiting walk and B's were folded into one successor, plan-2, that +// answers C's merge and B's and not A's. The carrier is in B's own walk, which waits for B's turn: B takes its +// turn there, and the one walk delivers C and B; A is covered once it is done. +func TestAMemberWhoseWalkCarriesTheSupersededOneBeforeItTakesItsTurn(t *testing.T) { + s := newShape445() + s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, + WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} + s.b.Walk = &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", + Branch: "main", Created: at445.Add(4 * time.Minute)} + two := ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", + Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}, {Repository: lab445, Commit: mergedB}}}) + two.Tiers = [][]string{modules445, {"mesh-lab"}} + two.Modules["mesh-lab"] = &WalkModule{} + s.walks = []Walk{s.walks[0], two} + w := s.open(t) + letGo445(t, w, "plan-2") + if s := w.state(cID); s != Delivering { + t.Fatalf("C is %s", s) + } + w.forge.contains[mergedA+" in "+mergedC] = true + end445(w, "plan-2", walkDone) + g := w.h.groups[g445] + if state, why := GroupState(g, w.h.membersOf(g)); state != "delivered" { + t.Fatalf("the group reads %s (%s), want delivered", state, why) + } +} + +// A carrier superseded in turn hands the wait to the later merge that took it over: C's waiting walk is folded +// into D's, which answers only D; B waits for D, and goes once D's walk covers A. +func TestTheWaitFollowsACarrierSupersededInTurn(t *testing.T) { + s := newShape445() + s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, + WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} + s.walks[2] = ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", + Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}) + s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", Branch: "fix/later-still", + State: Ready, Since: at445.Add(6 * time.Minute), Created: at445.Add(-time.Hour), MergedAs: mergedD, + MergedAt: at445.Add(6 * time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}}} + w := s.open(t) + // C is a lone delivery at its turn; the fake controller lets plan-2 go. It is folded all the same, as the + // review's shape has it, before it started. + two := w.ctl.walk("plan-2") + two.Delivery.Go = nil + two.State, two.Note = walkSuperseded, "superseded at tier 0 by plan-3 ("+ctl445+" at 4d4d4d4d); planned there again" + w.h.WalkMoved(w.ctl.put(ctlWalk("plan-3", mergedD, at445.Add(7*time.Minute), walkBuilding, + &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}))) + w.h.WalkMoved(w.ctl.put(two)) + w.settleAll() + if s := w.state(cID); s != Superseded { + t.Fatalf("C is %s", s) + } + waits445(t, w, dID445) + if strings.Contains(w.delivery(bID).TurnAwaits, cID) { + t.Fatalf("B still waits for C: %q", w.delivery(bID).TurnAwaits) + } + w.forge.contains[mergedA+" in "+mergedD] = true + end445(w, "plan-3", walkDone) + if d := w.delivery(aID); d.CoveredBy == "" || !strings.Contains(d.CoveredBy, dID445) { + t.Fatalf("A is covered by %q", d.CoveredBy) + } + letGo445(t, w, "plan-lab") +} + +// A merged carrier still in an assembling batch has no state bound of its own: it is awaited for +// readyCarrierBound from its merge, and then B is stopped. +func TestACarrierInABatchIsAwaitedWithinTheWindowAndOneWalk(t *testing.T) { + s := newShape445() + s.c.State, s.c.Walk = Ready, nil + closes := at445.Add(3 * time.Minute) + s.walks[2] = Walk{ID: "plan-batch", Repository: ctl445, Branch: "main", Commit: mergedC, State: walkQueued, + Created: at445.Add(2 * time.Minute), Delivery: &WalkDelivery{Awaits: "mesh-delivery", Batch: &WalkBatch{ + ClosesAt: &closes, Behind: "plan-other"}, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}} + w := s.open(t) + waits445(t, w, cID) + if want := s.c.MergedAt.Add(readyCarrierBound); !w.delivery(bID).TurnAwaitsUntil.Equal(want) { + t.Fatalf("B waits until %v, want %v", w.delivery(bID).TurnAwaitsUntil, want) + } + w.later(readyCarrierBound) + w.settleAll() + stopped445(t, w, "no walk carried it") +} + +// A member whose every module a later delivered merge carried, the forge saying it contains the member's merge, +// is reached even before CoveredBy is written on it: never a stop with an empty reason. +func TestACarriedMemberNotYetMarkedCoveredIsReached(t *testing.T) { + s := newShape445() + s.c.State, s.c.Since = Delivered, at445.Add(9*time.Minute) + s.c.Walk.State = walkDone + for _, m := range modules445 { + s.c.Steps = append(s.c.Steps, Step{Machine: "novox", Module: m, State: StepPassed}, + Step{Machine: stepTheRest, Module: m, State: StepSent}) + } + s.a.ContainedIn = map[string]bool{mergedC: true} + s.walks = s.walks[:2] + store := newMemStore() + for _, d := range []*Delivery{s.a, s.b, s.c} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", + Check: &GroupCheck{Verdict: "pass", Commits: commitsOf([]*Delivery{s.a, s.b})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + w := newWorldOver(t, store, newFakeController(), newFakeForge(), at445.Add(10*time.Minute)) + w.h.mu.Lock() + gg := w.h.groups[g445] + if w.h.deliveries[aID].CoveredBy != "" { + t.Fatal("A is marked covered already: the case is not the one under test") + } + w.h.turning = map[string]*turnWant{} + w.h.groupTurns(gg, w.h.membersOf(gg)) + b := w.h.deliveries[bID] + turned := w.h.turning["plan-lab"] != nil && containsString(w.h.turning["plan-lab"].ids, bID) + w.h.mu.Unlock() + if b.State != Published || !turned { + t.Fatalf("B is %s, at its turn %v; want its turn taken", b.State, turned) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 5f9964fb..9221ff13 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -65,6 +65,11 @@ type Delivery struct { // (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316). CoveredBy string `json:"covered_by,omitempty"` Lifted string `json:"lifted,omitempty"` + // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, + // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, + // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. + TurnAwaits string `json:"turn_awaits,omitempty"` + TurnAwaitsUntil time.Time `json:"turn_awaits_until,omitempty"` // ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's // merge (gitea_contains): asked once per pair, and kept (ADR 0270). ContainedIn map[string]bool `json:"contained_in,omitempty"` diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index ce8fa550..b2740e75 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -501,6 +501,13 @@ func boundOf(d *Delivery) (Bound, bool) { if d.waitsForItsOwnWalk() { return rewalkBound, true } + if d.State == Published && d.TurnAwaits != "" && !d.TurnAwaitsUntil.IsZero() { + // Its turn waits for a later merge to carry a member before it: bounded by that carrier's own bound, and + // said with it, never the generic "waits for its turn or its word" (novox/hq issue 445). + b := Bounds[Published] + b.For, b.Says = max(b.For, d.TurnAwaitsUntil.Sub(boundSince(d))), d.TurnAwaits + return b, true + } b, bounded := Bounds[d.State] return b, bounded } From 15a24fb63ec0f40f1172102874256aa3cfc378c5 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 11 Oct 2026 03:46:32 +0200 Subject: [PATCH 3/3] mesh-delivery: share a carrier's walk only when it delivers the whole superseded member A turn given on one carrier's waiting walk let the next member go while another module of the member before it waited on a walk that could still fail, and the stop then took the shared walk down. The turn is now shared only when that one walk carries every module not yet carried, each by a merge the forge said contains the member's; an unknown answer is asked and waited for (novox/hq issue 445, re-review of #203). --- .../mesh-delivery/cmd/mesh-delivery/holder.go | 92 ++++++++++-- .../cmd/mesh-delivery/issue445_test.go | 135 +++++++++++++++--- .../mesh-delivery/cmd/mesh-delivery/model.go | 4 +- .../mesh-delivery/cmd/mesh-delivery/table.go | 2 +- 4 files changed, 201 insertions(+), 32 deletions(-) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index b75f537e..9223c804 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -1406,8 +1406,14 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { if d.State != Published || !d.LetGo.IsZero() { w = carryWait{} // only a member still waiting for its turn waits for a carrier } - if d.TurnAwaits != w.said || !d.TurnAwaitsUntil.Equal(w.until) { - d.TurnAwaits, d.TurnAwaitsUntil = w.said, w.until + var until *time.Time + if !w.until.IsZero() { + u := w.until + until = &u + } + if d.TurnAwaits != w.said || (d.TurnAwaitsUntil == nil) != (until == nil) || + until != nil && !d.TurnAwaitsUntil.Equal(*until) { + d.TurnAwaits, d.TurnAwaitsUntil = w.said, until h.keep(d) } } @@ -1446,6 +1452,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa // wait, as for a member still delivering (ADR 0239 decision 4: the next starts when the one before is // delivered). Only when none can carry it are they stopped, saying why (novox/hq issue 445). w := h.carryAwaited(d, h.Now()) + // A delivered carrier is asked of the forge here too: coverOf asks only once every module has a + // delivered carrier, and while another module waits on an open one, this answer is what is awaited. + for _, in := range w.ask { + if a := h.askContains(d, in, h.Now()); a != nil { + asks = append(asks, a) + } + } if w.covered() { continue // every module delivered by a later merge the forge says contains it: reached } @@ -1468,7 +1481,12 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa // the next member's own walk, which waits for that member's turn. Waiting there would wait for itself // until the carrier's bound. That walk delivers both, in the group's order among its tiers (decision 5), // so the members on it take their turn as members sharing one walk do; a member on another walk waits. - if shared := w.waitingWalk(); shared != "" && sharing == "" { + // It is shared only when that one walk carries every module of the member not yet carried, each by a + // merge the forge said contains the member's: a carrier of some modules on it, others elsewhere, would + // let the next member go on a walk that cannot deliver the member before it (review of #203). + shared, ask := h.sharedWalk(d, w, h.Now()) + asks = append(asks, ask...) + if shared != "" && sharing == "" { sharing = shared continue } @@ -1538,22 +1556,65 @@ const readyCarrierBound = 10*time.Minute + 2*time.Hour // the moment the wait ends at the latest; or why none can carry it. Neither means every module it moves was // delivered by a later merge the forge says contains it: covered. type carryWait struct { - by []*Delivery - said string - until time.Time - why string + by []*Delivery + // byModule are the carriers awaited for each module not yet carried. + byModule map[string][]*Delivery + said string + until time.Time + why string + // ask are the delivered carriers' merge commits the forge has not yet said contain the member's. + ask []string } func (w carryWait) covered() bool { return len(w.by) == 0 && w.why == "" } -// waitingWalk is the walk of an awaited carrier that has not started: one waiting for this owner's word. -func (w carryWait) waitingWalk() string { +// sharedWalk is the walk, not started and waiting for this owner's word, that carries every module of a superseded +// member not yet carried, each by a carrier the forge said contains the member's merge: the walk the members after +// it may take their turn on. Whether a carrier on it contains the merge, when not yet known, is asked of the forge, +// and the members wait for the answer within the carrier's bound (ADR 0270, novox/hq issue 445). +func (h *Holder) sharedWalk(d *Delivery, w carryWait, now time.Time) (string, []func()) { + var walks []string for _, c := range w.by { - if !c.State.Final() && c.Walk != nil && c.Walk.Waits && !c.Walk.Started() { - return c.Walk.ID + if !c.State.Final() && c.Walk != nil && c.Walk.Waits && !c.Walk.Started() && !containsString(walks, c.Walk.ID) { + walks = append(walks, c.Walk.ID) } } - return "" + sort.Strings(walks) + var asks []func() + for _, walk := range walks { + every, said := true, true + var unknown []string + for _, carriers := range w.byModule { + on, yes := false, false + for _, c := range carriers { + if c.State.Final() || c.Walk == nil || c.Walk.ID != walk || !c.Walk.Waits || c.Walk.Started() { + continue + } + on = true + contained, known := d.ContainedIn[c.MergedAs] + switch { + case known && contained: + yes = true + case !known && !containsString(unknown, c.MergedAs): + unknown = append(unknown, c.MergedAs) + } + } + every = every && on + said = said && yes + } + if !every { + continue // a module is carried elsewhere, or by nothing on this walk: it cannot deliver the member + } + if said { + return walk, asks + } + for _, in := range unknown { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + return "", asks } // carryAwaited says which later merges into a superseded member's trunk may still carry every module it moves — @@ -1593,6 +1654,9 @@ modules: not = append(not, fmt.Sprintf("the forge did not say within %s whether %s contains its merge", containsGrace, c.ID)) continue } + if !containsString(w.ask, c.MergedAs) { + w.ask = append(w.ask, c.MergedAs) + } case c.State.Final() || c.State == Held: continue // ended without carrying it, or held for a person: no carrier to wait for case c.State == Ready: @@ -1612,6 +1676,10 @@ modules: } } found = true + if w.byModule == nil { + w.byModule = map[string][]*Delivery{} + } + w.byModule[module] = append(w.byModule[module], c) if until.After(latest) { latest = until } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go index 7215ca0f..89ba0119 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go @@ -35,6 +35,7 @@ type shape445 struct { a, b, c *Delivery extra []*Delivery walks []Walk + forge map[string]bool // what the forge says of containment from the start } func newShape445() *shape445 { @@ -102,7 +103,11 @@ func (s *shape445) open(t *testing.T) *world { for _, w := range s.walks { ctlr.put(w) } - w := newWorldOver(t, store, ctlr, newFakeForge(), at445.Add(10*time.Minute)) + forge := newFakeForge() + for k, v := range s.forge { + forge.contains[k] = v + } + w := newWorldOver(t, store, ctlr, forge, at445.Add(10*time.Minute)) if err := w.h.Reconcile(); err != nil { t.Fatal(err) } @@ -128,7 +133,7 @@ func waits445(t *testing.T, w *world, on string) { if d.State != Published || containsString(w.ctl.delivers, "plan-lab") { t.Fatalf("B is %s, let go %v; want it published and waiting while %s can still carry A", d.State, w.ctl.delivers, on) } - if !strings.Contains(d.TurnAwaits, on) || !strings.Contains(d.TurnAwaits, aID) || d.TurnAwaitsUntil.IsZero() { + if !strings.Contains(d.TurnAwaits, on) || !strings.Contains(d.TurnAwaits, aID) || d.TurnAwaitsUntil == nil { t.Fatalf("B says it waits for %q until %v; want it to name A and %s", d.TurnAwaits, d.TurnAwaitsUntil, on) } g := w.h.groups[g445] @@ -227,22 +232,13 @@ func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { // answers C's merge and B's and not A's. The carrier is in B's own walk, which waits for B's turn: B takes its // turn there, and the one walk delivers C and B; A is covered once it is done. func TestAMemberWhoseWalkCarriesTheSupersededOneBeforeItTakesItsTurn(t *testing.T) { - s := newShape445() - s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, - WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} - s.b.Walk = &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", - Branch: "main", Created: at445.Add(4 * time.Minute)} - two := ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", - Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}, {Repository: lab445, Commit: mergedB}}}) - two.Tiers = [][]string{modules445, {"mesh-lab"}} - two.Modules["mesh-lab"] = &WalkModule{} - s.walks = []Walk{s.walks[0], two} + s := foldShape() + s.forge = map[string]bool{mergedA + " in " + mergedC: true} w := s.open(t) letGo445(t, w, "plan-2") if s := w.state(cID); s != Delivering { t.Fatalf("C is %s", s) } - w.forge.contains[mergedA+" in "+mergedC] = true end445(w, "plan-2", walkDone) g := w.h.groups[g445] if state, why := GroupState(g, w.h.membersOf(g)); state != "delivered" { @@ -280,6 +276,8 @@ func TestTheWaitFollowsACarrierSupersededInTurn(t *testing.T) { } w.forge.contains[mergedA+" in "+mergedD] = true end445(w, "plan-3", walkDone) + w.later(2 * time.Minute) // the forge, asked while it could not say, is asked again a minute on + w.settleAll() if d := w.delivery(aID); d.CoveredBy == "" || !strings.Contains(d.CoveredBy, dID445) { t.Fatalf("A is covered by %q", d.CoveredBy) } @@ -297,12 +295,23 @@ func TestACarrierInABatchIsAwaitedWithinTheWindowAndOneWalk(t *testing.T) { ClosesAt: &closes, Behind: "plan-other"}, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}} w := s.open(t) waits445(t, w, cID) - if want := s.c.MergedAt.Add(readyCarrierBound); !w.delivery(bID).TurnAwaitsUntil.Equal(want) { - t.Fatalf("B waits until %v, want %v", w.delivery(bID).TurnAwaitsUntil, want) + // The merge window at its longest, then the one open walk the batch may queue behind (ADR 0276 decisions 1, + // 4 and 9). + if readyCarrierBound != 10*time.Minute+Bounds[Delivering].For { + t.Fatalf("a ready carrier is awaited %s, want the window's 10 minutes and one walk's %s", readyCarrierBound, + Bounds[Delivering].For) } - w.later(readyCarrierBound) + until := s.c.MergedAt.Add(readyCarrierBound) + if got := w.delivery(bID).TurnAwaitsUntil; got == nil || !got.Equal(until) { + t.Fatalf("B waits until %v, want %v", got, until) + } + // Within the bound, past every state bound a ready delivery could be read against, B still waits. + w.later(until.Add(-time.Minute).Sub(w.now)) w.settleAll() - stopped445(t, w, "no walk carried it") + waits445(t, w, cID) + w.later(2 * time.Minute) + w.settleAll() + stopped445(t, w, "no walk carried it within "+readyCarrierBound.String()) } // A member whose every module a later delivered merge carried, the forge saying it contains the member's merge, @@ -343,3 +352,95 @@ func TestACarriedMemberNotYetMarkedCoveredIsReached(t *testing.T) { t.Fatalf("B is %s, at its turn %v; want its turn taken", b.State, turned) } } + +// foldShape is the review's shape: B and C on one walk, plan-2, not started, waiting for mesh-delivery's word. +func foldShape() *shape445 { + s := newShape445() + s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, + WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} + s.b.Walk = &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", + Branch: "main", Created: at445.Add(4 * time.Minute)} + two := ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", + Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}, {Repository: lab445, Commit: mergedB}}}) + two.Tiers = [][]string{modules445, {"mesh-lab"}} + two.Modules["mesh-lab"] = &WalkModule{} + s.walks = []Walk{s.walks[0], two} + return s +} + +// The turn on the carrier's walk is given only on the forge's word that the carrier contains A's merge: not yet +// said, B waits and the forge is asked; said "no", C is no carrier and B is stopped, never let go on plan-2. +func TestASharedTurnWaitsForTheForgesWord(t *testing.T) { + t.Run("not yet said: B waits and the forge is asked", func(t *testing.T) { + w := foldShape().open(t) + waits445(t, w, cID) + if containsString(w.ctl.delivers, "plan-2") { + t.Fatalf("plan-2 was let go %v before the forge said C contains A", w.ctl.delivers) + } + w.forge.mu.Lock() + asked := containsString(w.forge.containsAsks, mergedA+" in "+mergedC) + w.forge.mu.Unlock() + if !asked { + t.Fatal("the forge was not asked whether C's merge contains A's") + } + w.forge.contains[mergedA+" in "+mergedC] = true + w.later(2 * time.Minute) + w.settleAll() + letGo445(t, w, "plan-2") + }) + t.Run("said no: no shared turn", func(t *testing.T) { + s := foldShape() + s.forge = map[string]bool{mergedA + " in " + mergedC: false} + w := s.open(t) + if d := w.delivery(bID); d.State == Delivering || d.LetGo != (time.Time{}) { + t.Fatalf("B was given its turn on plan-2 (%s, let go %v)", d.State, d.LetGo) + } + stopped445(t, w, "does not contain its merge") + }) +} + +// Two carriers for A: C on B's own walk carries one module, D on a walk of its own the other. B is not given its +// turn on C's walk, which cannot deliver A whole: it waits for D's walk, goes once D covers its module, and is +// stopped, never delivering, when D's walk fails (re-review of #203). +func TestASharedTurnNeedsEveryModuleOnTheOneWalk(t *testing.T) { + two := func() *shape445 { + s := foldShape() + s.a.Plan = planMoving(ctl445, "mesh-controller", "route-proxy") + s.c.Plan = planMoving(ctl445, "mesh-controller") + s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", + Branch: "fix/other", State: Delivering, Since: at445.Add(5 * time.Minute), Created: at445.Add(-time.Hour), + MergedAs: mergedD, MergedAt: at445.Add(90 * time.Second), Plan: planMoving(ctl445, "route-proxy"), + Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-other", Commit: mergedD, State: walkRolling, + WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at445.Add(4 * time.Minute)}}} + went := at445.Add(5 * time.Minute) + other := ctlWalk("plan-other", mergedD, at445.Add(4*time.Minute), walkRolling, &WalkDelivery{ + Awaits: "mesh-delivery", Go: &went, By: byOwner, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}) + other.Modules = map[string]*WalkModule{"route-proxy": {}} + s.walks = append(s.walks, other) + s.forge = map[string]bool{mergedA + " in " + mergedC: true, mergedA + " in " + mergedD: true} + return s + } + t.Run("D's walk open: B waits for both", func(t *testing.T) { + w := two().open(t) + waits445(t, w, dID445) + if !strings.Contains(w.delivery(bID).TurnAwaits, cID) || containsString(w.ctl.delivers, "plan-2") { + t.Fatalf("B is %s, let go %v, waiting for %q", w.state(bID), w.ctl.delivers, w.delivery(bID).TurnAwaits) + } + }) + t.Run("D's walk delivered: B goes on C's walk", func(t *testing.T) { + w := two().open(t) + end445(w, "plan-other", walkDone) + letGo445(t, w, "plan-2") + }) + t.Run("D's walk failed: B is stopped, never delivering", func(t *testing.T) { + w := two().open(t) + end445(w, "plan-other", walkFailed) + d := w.delivery(bID) + for _, tr := range d.Transitions { + if tr.To == Delivering { + t.Fatalf("B was delivering before it was stopped: %+v", d.Transitions) + } + } + stopped445(t, w, "route-proxy") + }) +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 9221ff13..25ec79e1 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -68,8 +68,8 @@ type Delivery struct { // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. - TurnAwaits string `json:"turn_awaits,omitempty"` - TurnAwaitsUntil time.Time `json:"turn_awaits_until,omitempty"` + TurnAwaits string `json:"turn_awaits,omitempty"` + TurnAwaitsUntil *time.Time `json:"turn_awaits_until,omitempty"` // ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's // merge (gitea_contains): asked once per pair, and kept (ADR 0270). ContainedIn map[string]bool `json:"contained_in,omitempty"` diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index b2740e75..ad55d7ff 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -501,7 +501,7 @@ func boundOf(d *Delivery) (Bound, bool) { if d.waitsForItsOwnWalk() { return rewalkBound, true } - if d.State == Published && d.TurnAwaits != "" && !d.TurnAwaitsUntil.IsZero() { + if d.State == Published && d.TurnAwaits != "" && d.TurnAwaitsUntil != nil { // Its turn waits for a later merge to carry a member before it: bounded by that carrier's own bound, and // said with it, never the generic "waits for its turn or its word" (novox/hq issue 445). b := Bounds[Published]