diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 360746eb..9223c804 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -1393,8 +1393,35 @@ func (h *Holder) inOrder(g *Group, members []*Delivery) []Member { // groupTurns walks a delivering group in its order: the next member is let go only when every member before // it is delivered; a member that failed or was stopped stops every member after it, naming it. What was // delivered before it stays, each judged at its own gate (ADR 0239 decision 4); the failed member's builds -// were put back at its gate by the controller. +// were put back at its gate by the controller. A superseded member counts as delivered once a later delivered +// walk carried it, holds the members after it while a later merge can still carry it, and stops them, saying +// why, only when none can (novox/hq issue 445). func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { + waits := map[string]carryWait{} + asks := h.turnsOf(g, members, waits) + // What each member's turn waits on a carrier for is kept on it, and cleared once it no longer does, so `show` + // and `stalled` name the carrier and until when (ADR 0276 decision 9, novox/hq issue 445). + for _, d := range members { + w := waits[d.ID] + if d.State != Published || !d.LetGo.IsZero() { + w = carryWait{} // only a member still waiting for its turn waits for a carrier + } + var until *time.Time + if !w.until.IsZero() { + u := w.until + until = &u + } + if d.TurnAwaits != w.said || (d.TurnAwaitsUntil == nil) != (until == nil) || + until != nil && !d.TurnAwaitsUntil.Equal(*until) { + d.TurnAwaits, d.TurnAwaitsUntil = w.said, until + h.keep(d) + } + } + return asks +} + +// turnsOf is groupTurns' walk through the order; waits receives, by member, the carrier its turn waits for. +func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWait) []func() { ordered := h.inOrder(g, members) byID := map[string]*Delivery{} for _, d := range members { @@ -1409,18 +1436,62 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { sharing := "" for i, m := range ordered { d := byID[m.ID] - if d.State.Reached() { + // A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a + // covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq + // issues 442 and 445). + if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" { continue } if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { return asks } switch { - case d.State == Failed || d.State == Stopped || d.State == Superseded: - // A superseded member is an end here, and it stops the members after it even when a later delivered - // walk carried it. GroupState counts such a member as reached (novox/hq issue 442), so for a carried - // member the two do not agree: the members after it are stopped all the same, and the group then reads - // stopped for them. That gap is novox/hq issue 445, not closed here. + case d.State == Superseded: + // Superseded is no failure: the merge that took its walk over may still deliver its change. While a later + // merge into its trunk that can carry it is open, within that merge's own bound, the members after it + // wait, as for a member still delivering (ADR 0239 decision 4: the next starts when the one before is + // delivered). Only when none can carry it are they stopped, saying why (novox/hq issue 445). + w := h.carryAwaited(d, h.Now()) + // A delivered carrier is asked of the forge here too: coverOf asks only once every module has a + // delivered carrier, and while another module waits on an open one, this answer is what is awaited. + for _, in := range w.ask { + if a := h.askContains(d, in, h.Now()); a != nil { + asks = append(asks, a) + } + } + if w.covered() { + continue // every module delivered by a later merge the forge says contains it: reached + } + if w.why != "" { + for _, later := range ordered[i+1:] { + if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, + d.ID, w.why)); a != nil { + asks = append(asks, a) + } + } + return asks + } + w.said = fmt.Sprintf("its turn in group %s waits for %s, superseded, to be carried by %s, until %s", g.ID, + d.ID, w.said, w.until.UTC().Format(time.RFC3339)) + for _, later := range ordered[i+1:] { + waits[later.ID] = w + } + // Under ADR 0276 a member is superseded when its waiting walk was folded into a successor that does not + // answer its merge, and the member after it was folded into the same successor: the carrier is then in + // the next member's own walk, which waits for that member's turn. Waiting there would wait for itself + // until the carrier's bound. That walk delivers both, in the group's order among its tiers (decision 5), + // so the members on it take their turn as members sharing one walk do; a member on another walk waits. + // It is shared only when that one walk carries every module of the member not yet carried, each by a + // merge the forge said contains the member's: a carrier of some modules on it, others elsewhere, would + // let the next member go on a walk that cannot deliver the member before it (review of #203). + shared, ask := h.sharedWalk(d, w, h.Now()) + asks = append(asks, ask...) + if shared != "" && sharing == "" { + sharing = shared + continue + } + return asks + case d.State == Failed || d.State == Stopped: for _, later := range ordered[i+1:] { if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { asks = append(asks, a) @@ -1458,6 +1529,7 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { } if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil { asks, sharing = append(asks, a), d.Walk.ID + delete(waits, d.ID) continue } return asks @@ -1468,6 +1540,188 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { return asks } +// containsGrace is how long a later merge whose walk was delivered may still be awaited as a superseded member's +// carrier while the forge has not said whether its merge commit contains the member's: as long as a published +// delivery may wait for its turn (Bounds[Published]). A forge that cannot answer means "not yet covered" (ADR +// 0270), and without this bound the members after it would wait on that answer for ever (novox/hq issue 445). +const containsGrace = 30 * time.Minute + +// readyCarrierBound is how long a merged carrier with no walk yet may be awaited from its merge. Its state, ready, +// has no bound of its own: it waits in a batch, which closes at the latest ten minutes after its first merge +// (ADR 0276 decision 1, `merge-window-at-most`), and then behind at most the one open walk, within that walk's +// bound (decisions 4 and 9). A merge no walk answers is held after `unwalked`, and a held one is no carrier. +const readyCarrierBound = 10*time.Minute + 2*time.Hour + +// carryWait is what a superseded member's group waits for: the later merges that may still carry it, said, and +// the moment the wait ends at the latest; or why none can carry it. Neither means every module it moves was +// delivered by a later merge the forge says contains it: covered. +type carryWait struct { + by []*Delivery + // byModule are the carriers awaited for each module not yet carried. + byModule map[string][]*Delivery + said string + until time.Time + why string + // ask are the delivered carriers' merge commits the forge has not yet said contain the member's. + ask []string +} + +func (w carryWait) covered() bool { return len(w.by) == 0 && w.why == "" } + +// sharedWalk is the walk, not started and waiting for this owner's word, that carries every module of a superseded +// member not yet carried, each by a carrier the forge said contains the member's merge: the walk the members after +// it may take their turn on. Whether a carrier on it contains the merge, when not yet known, is asked of the forge, +// and the members wait for the answer within the carrier's bound (ADR 0270, novox/hq issue 445). +func (h *Holder) sharedWalk(d *Delivery, w carryWait, now time.Time) (string, []func()) { + var walks []string + for _, c := range w.by { + if !c.State.Final() && c.Walk != nil && c.Walk.Waits && !c.Walk.Started() && !containsString(walks, c.Walk.ID) { + walks = append(walks, c.Walk.ID) + } + } + sort.Strings(walks) + var asks []func() + for _, walk := range walks { + every, said := true, true + var unknown []string + for _, carriers := range w.byModule { + on, yes := false, false + for _, c := range carriers { + if c.State.Final() || c.Walk == nil || c.Walk.ID != walk || !c.Walk.Waits || c.Walk.Started() { + continue + } + on = true + contained, known := d.ContainedIn[c.MergedAs] + switch { + case known && contained: + yes = true + case !known && !containsString(unknown, c.MergedAs): + unknown = append(unknown, c.MergedAs) + } + } + every = every && on + said = said && yes + } + if !every { + continue // a module is carried elsewhere, or by nothing on this walk: it cannot deliver the member + } + if said { + return walk, asks + } + for _, in := range unknown { + if a := h.askContains(d, in, now); a != nil { + asks = append(asks, a) + } + } + } + return "", asks +} + +// carryAwaited says which later merges into a superseded member's trunk may still carry every module it moves — +// the cover GroupState counts as reached (ADR 0270's covered test, novox/hq issue 442) — and until when, or, +// when none can, why. A later merge of the same repository into the same trunk on a passing check, whose plan +// moves the module and whose merge commit the forge has not said leaves this one out, can carry it while it is +// merged and not yet ended, not held for a person, and within its bound (ADR 0239 decision 9, ADR 0276 decision +// 9: every wait has a bound): its state's, or readyCarrierBound from its merge while no walk carries it yet; once +// delivered, while the forge has not yet said whether it contains this merge, within containsGrace. A carrier +// superseded in turn ends; the later merge that took it over is a carrier of its own (novox/hq issue 445). +func (h *Holder) carryAwaited(d *Delivery, now time.Time) carryWait { + if d.MergedAs == "" || d.MergedAt.IsZero() || d.Plan == nil || len(d.Plan.Moved) == 0 { + return carryWait{why: "nothing records which modules it moves, so no later merge can carry it"} + } + var w carryWait + seen := map[string]bool{} +modules: + for _, module := range d.Plan.Moved { + found := false + var latest time.Time // the last moment a carrier of this module may still be awaited + var not []string + for _, c := range h.deliveries { + if !mayCarry(c, d, module) { + continue + } + contained, known := d.ContainedIn[c.MergedAs] + var until time.Time + switch { + case known && !contained: + not = append(not, c.ID+" does not contain its merge") + continue + case c.State == Delivered && carries(c, d, module) && known: + continue modules // delivered and contained: this module is carried + case c.State == Delivered && carries(c, d, module): + until = c.Since.Add(containsGrace) + if now.After(until) { + not = append(not, fmt.Sprintf("the forge did not say within %s whether %s contains its merge", containsGrace, c.ID)) + continue + } + if !containsString(w.ask, c.MergedAs) { + w.ask = append(w.ask, c.MergedAs) + } + case c.State.Final() || c.State == Held: + continue // ended without carrying it, or held for a person: no carrier to wait for + case c.State == Ready: + until = c.MergedAt.Add(readyCarrierBound) + if now.After(until) { + not = append(not, fmt.Sprintf("%s is merged and no walk carried it within %s", c.ID, readyCarrierBound)) + continue + } + default: + b, bounded := boundOf(c) + if pastBound(c, now) { + not = append(not, fmt.Sprintf("%s is past its bound (%s %s)", c.ID, c.State, b.For)) + continue + } + if until = now.Add(Bounds[Delivering].For); bounded && b.For > 0 { + until = boundSince(c).Add(b.For) + } + } + found = true + if w.byModule == nil { + w.byModule = map[string][]*Delivery{} + } + w.byModule[module] = append(w.byModule[module], c) + if until.After(latest) { + latest = until + } + if !seen[c.ID] { + seen[c.ID] = true + w.by = append(w.by, c) + } + } + if !found { + why := fmt.Sprintf("no later merge into %s can carry %s", d.Base, module) + if len(not) > 0 { + sort.Strings(not) + why += ": " + strings.Join(not, "; ") + } + return carryWait{why: why} + } + // The wait ends when the first module runs out of carriers, whatever the others still have. + if w.until.IsZero() || latest.Before(w.until) { + w.until = latest + } + } + sort.Slice(w.by, func(i, j int) bool { return w.by[i].ID < w.by[j].ID }) + var said []string + for _, c := range w.by { + s := fmt.Sprintf("%s (%s", c.ID, c.State) + if c.Walk != nil { + s += ", walk " + c.Walk.ID + } + said = append(said, s+")") + } + w.said = strings.Join(said, ", ") + return w +} + +// mayCarry is whether c is a later merge that could carry module for d, by what is known before its walk ends: of +// the same repository, merged later into the same trunk, on its own passing check, its plan moving the module. +func mayCarry(c, d *Delivery, module string) bool { + return c.ID != d.ID && strings.EqualFold(c.Repository, d.Repository) && strings.EqualFold(c.Base, d.Base) && + c.MergedAs != "" && !sameCommit(c.MergedAs, d.MergedAs) && c.MergedAt.After(d.MergedAt) && c.Check.Passes() && + c.Plan != nil && slices.Contains(c.Plan.Moved, module) +} + // stopInGroup stops a member after one that failed, and its walk if it has one open. func (h *Holder) stopInGroup(d *Delivery, why string) func() { if d == nil || d.State.Final() { @@ -1499,8 +1753,8 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() { // 0270's covered test); a closed group whose members all ended is then delivered, and otherwise stopped, naming // the member. A group with nothing in progress never reads delivering. // -// groupTurns does not yet agree for a carried member: it stops the members after any superseded one, carried or -// not, and a stopped member makes the group read stopped here. That gap is novox/hq issue 445. +// groupTurns agrees: the members after a carried member take their turn, and they wait while a later merge can still +// carry it (novox/hq issue 445). func GroupState(g *Group, members []*Delivery) (string, string) { count := map[State]int{} reached, ended := 0, 0 diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go new file mode 100644 index 00000000..89ba0119 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue445_test.go @@ -0,0 +1,446 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 445: in a delivery group ordered [A, B], A's walk was superseded by the walk of a later merge C +// of A's repository that does not answer A's merge. groupTurns stopped B at once, although C's walk could still +// carry A's modules and A then counts as reached (ADR 0270 decision 2, issue 442). While a later merge that can +// carry A is open, B waits, saying for which merge and until when; once that merge's walk is delivered and covers +// A, B takes its turn; when C's walk is B's own (ADR 0276 folds both into one successor), B takes its turn on it; +// when no later merge can carry A, or the one that could is past its own bound, B is stopped, saying why. +// +// The shape is the live group fix/assign-says-why-a-module-is-not-there as mesh-delivery showed it on +// 2026-10-11 (issue 442's test): the controller's member superseded first in the order, the lab's after it. + +const ( + g445 = "fix/assign-says-why-a-module-is-not-there" + ctl445, lab445 = "novox/mesh-controller", "novox/mesh-lab" + mergedA, mergedB, mergedC = "a5f53ef9ebdf3650", "77777777aaaaaaaa", "3f68a49500000000" + mergedD = "4d4d4d4d00000000" + headA, headB, headC, headD = "8adb7f1a051941e9", "33e63ee5b47a7552", "3f68a495c0c0c0c0", "4d4d4d4dc0c0c0c0" +) + +var ( + at445 = time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) + modules445 = []string{"build-agent", "mesh-controller", "route-proxy"} + aID, bID, cID, dID445 = IDOf(ctl445, headA), IDOf(lab445, headB), IDOf(ctl445, headC), IDOf(ctl445, headD) +) + +// shape445 is A superseded, B published on its walk, and C, the later merge of A's repository, as each case sets it. +type shape445 struct { + a, b, c *Delivery + extra []*Delivery + walks []Walk + forge map[string]bool // what the forge says of containment from the start +} + +func newShape445() *shape445 { + at := at445 + s := &shape445{} + s.a = &Delivery{ID: aID, Repository: ctl445, Commit: headA, Number: 109, Base: "main", Branch: g445, Group: g445, + State: Superseded, Since: at.Add(4 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedA, MergedAt: at, + Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-a5f53e", Commit: mergedA, State: walkSuperseded, Branch: "main", + Note: "superseded at tier 0 by plan-later (" + ctl445 + " at 3f68a495); planned there again"}, + Transitions: []Transition{{At: at.Add(4 * time.Minute), From: Published, To: Superseded, Event: EvSuperseded, + Why: "a newer delivery to the same trunk took over its walk"}}} + s.b = &Delivery{ID: bID, Repository: lab445, Commit: headB, Number: 117, Base: "main", Branch: g445, Group: g445, + State: Published, Since: at.Add(3 * time.Minute), Created: at.Add(-2 * time.Hour), MergedAs: mergedB, + MergedAt: at.Add(2 * time.Minute), Plan: planMoving(lab445, "mesh-lab"), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-lab", Commit: mergedB, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", + Branch: "main", Created: at.Add(3 * time.Minute)}} + s.c = &Delivery{ID: cID, Repository: ctl445, Commit: headC, Number: 110, Base: "main", Branch: "fix/later", + State: Delivering, Since: at.Add(5 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedC, + MergedAt: at.Add(time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, + Walk: &WalkSeen{ID: "plan-later", Commit: mergedC, State: walkRolling, WaitedFor: "mesh-delivery", GoBy: byOwner, + Branch: "main", Created: at.Add(4 * time.Minute)}} + went := at.Add(5 * time.Minute) + s.walks = []Walk{ + {ID: "plan-a5f53e", Repository: ctl445, Branch: "main", Commit: mergedA, Created: at.Add(30 * time.Second), + State: walkSuperseded, Note: s.a.Walk.Note, Modules: map[string]*WalkModule{}}, + {ID: "plan-lab", Repository: lab445, Branch: "main", Commit: mergedB, Created: at.Add(3 * time.Minute), + State: walkBuilding, Tiers: [][]string{{"mesh-lab"}}, Modules: map[string]*WalkModule{"mesh-lab": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}, + ctlWalk("plan-later", mergedC, at.Add(4*time.Minute), walkRolling, &WalkDelivery{Awaits: "mesh-delivery", + Go: &went, By: byOwner}), + } + return s +} + +// ctlWalk is a walk of the controller's repository at a merge commit, moving A's modules. +func ctlWalk(id, commit string, created time.Time, state string, wd *WalkDelivery) Walk { + w := Walk{ID: id, Repository: ctl445, Branch: "main", Commit: commit, Created: created, State: state, + Tiers: [][]string{modules445}, Modules: map[string]*WalkModule{}, Delivery: wd} + for _, m := range modules445 { + w.Modules[m] = &WalkModule{} + } + return w +} + +// open is the world over the shape, read back and ticked ten minutes after A's merge. +func (s *shape445) open(t *testing.T) *world { + t.Helper() + store := newMemStore() + for _, d := range append([]*Delivery{s.a, s.b, s.c}, s.extra...) { + if d == nil { + continue + } + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", + OrderOf: map[string]string{}, Check: &GroupCheck{Asked: "check-1", At: at445.Add(-time.Hour), Verdict: "pass", + Summary: "both compose", Commits: commitsOf([]*Delivery{s.a, s.b})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + ctlr := newFakeController() + for _, w := range s.walks { + ctlr.put(w) + } + forge := newFakeForge() + for k, v := range s.forge { + forge.contains[k] = v + } + w := newWorldOver(t, store, ctlr, forge, at445.Add(10*time.Minute)) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + return w +} + +// end ends a walk in the state given, every module sent to every machine running it. +func end445(w *world, walk, state string) { + r := w.ctl.walk(walk) + sent := w.now + for m := range r.Modules { + r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} + } + r.State = state + w.h.WalkMoved(w.ctl.put(r)) + w.settleAll() +} + +func waits445(t *testing.T, w *world, on string) { + t.Helper() + d := w.delivery(bID) + if d.State != Published || containsString(w.ctl.delivers, "plan-lab") { + t.Fatalf("B is %s, let go %v; want it published and waiting while %s can still carry A", d.State, w.ctl.delivers, on) + } + if !strings.Contains(d.TurnAwaits, on) || !strings.Contains(d.TurnAwaits, aID) || d.TurnAwaitsUntil == nil { + t.Fatalf("B says it waits for %q until %v; want it to name A and %s", d.TurnAwaits, d.TurnAwaitsUntil, on) + } + g := w.h.groups[g445] + if state, why := GroupState(g, w.h.membersOf(g)); state != "delivering" { + t.Fatalf("the group reads %s (%s), want delivering", state, why) + } +} + +func stopped445(t *testing.T, w *world, says string) { + t.Helper() + d := w.delivery(bID) + if d.State != Stopped || len(d.Transitions) == 0 { + t.Fatalf("B is %s, want stopped", d.State) + } + why := d.Transitions[len(d.Transitions)-1].Why + if !strings.Contains(why, aID) || !strings.Contains(why, says) { + t.Fatalf("B was stopped saying %q, want it to name %s and say %q", why, aID, says) + } + if containsString(w.ctl.delivers, "plan-lab") || d.TurnAwaits != "" { + t.Fatalf("B's walk was let go %v, or B still says it waits: %q", w.ctl.delivers, d.TurnAwaits) + } +} + +func letGo445(t *testing.T, w *world, walk string) { + t.Helper() + if d := w.delivery(bID); d.State != Delivering || !containsString(w.ctl.delivers, walk) || d.TurnAwaits != "" { + t.Fatalf("B is %s, let go %v, waiting for %q; want its turn taken on %s", d.State, w.ctl.delivers, d.TurnAwaits, walk) + } +} + +// C's walk runs on its own: B waits for it, says so, and goes on or stops by what comes of it. +func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { + t.Run("a later walk that can carry A is open: B waits, naming it", func(t *testing.T) { + w := newShape445().open(t) + waits445(t, w, cID) + // Past a published delivery's own 30 minutes, B is not stalled for "its turn or its word": its bound is + // C's, and what `stalled` would say names C. + w.later(Bounds[Published].For + 5*time.Minute) + w.settleAll() + waits445(t, w, cID) + for _, l := range w.h.Stalled() { + if l.ID == bID { + t.Fatalf("B is listed stalled while C's walk is within its bound: %+v", l) + } + } + if b, _ := boundOf(w.delivery(bID)); !strings.Contains(b.Says, cID) || !strings.Contains(b.Says, "until") { + t.Fatalf("B's bound says %q", b.Says) + } + }) + t.Run("that walk delivered and covers A: B takes its turn", func(t *testing.T) { + w := newShape445().open(t) + w.forge.contains[mergedA+" in "+mergedC] = true + end445(w, "plan-later", walkDone) + if s := w.state(cID); s != Delivered { + t.Fatalf("C is %s", s) + } + if d := w.delivery(aID); d.State != Superseded || d.CoveredBy == "" { + t.Fatalf("A is %s, covered by %q", d.State, d.CoveredBy) + } + letGo445(t, w, "plan-lab") + }) + t.Run("the forge says the later merge does not contain A: B is stopped", func(t *testing.T) { + w := newShape445().open(t) + w.forge.contains[mergedA+" in "+mergedC] = false + end445(w, "plan-later", walkDone) + stopped445(t, w, "can carry") + }) + t.Run("the later walk failed: B is stopped", func(t *testing.T) { + w := newShape445().open(t) + end445(w, "plan-later", walkFailed) + if s := w.state(cID); s != Failed { + t.Fatalf("C is %s", s) + } + stopped445(t, w, "can carry") + }) + t.Run("the later walk stays open past its own bound: B is stopped", func(t *testing.T) { + w := newShape445().open(t) + w.later(Bounds[Delivering].For - 10*time.Minute) + w.settleAll() + waits445(t, w, cID) + w.later(10 * time.Minute) + w.settleAll() + stopped445(t, w, "past its bound") + }) + t.Run("the forge never says whether the delivered walk contains A: B is stopped after the grace", func(t *testing.T) { + w := newShape445().open(t) + end445(w, "plan-later", walkDone) + waits445(t, w, cID) + w.later(containsGrace + time.Minute) + w.settleAll() + stopped445(t, w, "did not say") + }) +} + +// The review's shape of #203 (ADR 0276): A's waiting walk and B's were folded into one successor, plan-2, that +// answers C's merge and B's and not A's. The carrier is in B's own walk, which waits for B's turn: B takes its +// turn there, and the one walk delivers C and B; A is covered once it is done. +func TestAMemberWhoseWalkCarriesTheSupersededOneBeforeItTakesItsTurn(t *testing.T) { + s := foldShape() + s.forge = map[string]bool{mergedA + " in " + mergedC: true} + w := s.open(t) + letGo445(t, w, "plan-2") + if s := w.state(cID); s != Delivering { + t.Fatalf("C is %s", s) + } + end445(w, "plan-2", walkDone) + g := w.h.groups[g445] + if state, why := GroupState(g, w.h.membersOf(g)); state != "delivered" { + t.Fatalf("the group reads %s (%s), want delivered", state, why) + } +} + +// A carrier superseded in turn hands the wait to the later merge that took it over: C's waiting walk is folded +// into D's, which answers only D; B waits for D, and goes once D's walk covers A. +func TestTheWaitFollowsACarrierSupersededInTurn(t *testing.T) { + s := newShape445() + s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, + WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} + s.walks[2] = ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", + Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}) + s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", Branch: "fix/later-still", + State: Ready, Since: at445.Add(6 * time.Minute), Created: at445.Add(-time.Hour), MergedAs: mergedD, + MergedAt: at445.Add(6 * time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}}} + w := s.open(t) + // C is a lone delivery at its turn; the fake controller lets plan-2 go. It is folded all the same, as the + // review's shape has it, before it started. + two := w.ctl.walk("plan-2") + two.Delivery.Go = nil + two.State, two.Note = walkSuperseded, "superseded at tier 0 by plan-3 ("+ctl445+" at 4d4d4d4d); planned there again" + w.h.WalkMoved(w.ctl.put(ctlWalk("plan-3", mergedD, at445.Add(7*time.Minute), walkBuilding, + &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}))) + w.h.WalkMoved(w.ctl.put(two)) + w.settleAll() + if s := w.state(cID); s != Superseded { + t.Fatalf("C is %s", s) + } + waits445(t, w, dID445) + if strings.Contains(w.delivery(bID).TurnAwaits, cID) { + t.Fatalf("B still waits for C: %q", w.delivery(bID).TurnAwaits) + } + w.forge.contains[mergedA+" in "+mergedD] = true + end445(w, "plan-3", walkDone) + w.later(2 * time.Minute) // the forge, asked while it could not say, is asked again a minute on + w.settleAll() + if d := w.delivery(aID); d.CoveredBy == "" || !strings.Contains(d.CoveredBy, dID445) { + t.Fatalf("A is covered by %q", d.CoveredBy) + } + letGo445(t, w, "plan-lab") +} + +// A merged carrier still in an assembling batch has no state bound of its own: it is awaited for +// readyCarrierBound from its merge, and then B is stopped. +func TestACarrierInABatchIsAwaitedWithinTheWindowAndOneWalk(t *testing.T) { + s := newShape445() + s.c.State, s.c.Walk = Ready, nil + closes := at445.Add(3 * time.Minute) + s.walks[2] = Walk{ID: "plan-batch", Repository: ctl445, Branch: "main", Commit: mergedC, State: walkQueued, + Created: at445.Add(2 * time.Minute), Delivery: &WalkDelivery{Awaits: "mesh-delivery", Batch: &WalkBatch{ + ClosesAt: &closes, Behind: "plan-other"}, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}} + w := s.open(t) + waits445(t, w, cID) + // The merge window at its longest, then the one open walk the batch may queue behind (ADR 0276 decisions 1, + // 4 and 9). + if readyCarrierBound != 10*time.Minute+Bounds[Delivering].For { + t.Fatalf("a ready carrier is awaited %s, want the window's 10 minutes and one walk's %s", readyCarrierBound, + Bounds[Delivering].For) + } + until := s.c.MergedAt.Add(readyCarrierBound) + if got := w.delivery(bID).TurnAwaitsUntil; got == nil || !got.Equal(until) { + t.Fatalf("B waits until %v, want %v", got, until) + } + // Within the bound, past every state bound a ready delivery could be read against, B still waits. + w.later(until.Add(-time.Minute).Sub(w.now)) + w.settleAll() + waits445(t, w, cID) + w.later(2 * time.Minute) + w.settleAll() + stopped445(t, w, "no walk carried it within "+readyCarrierBound.String()) +} + +// A member whose every module a later delivered merge carried, the forge saying it contains the member's merge, +// is reached even before CoveredBy is written on it: never a stop with an empty reason. +func TestACarriedMemberNotYetMarkedCoveredIsReached(t *testing.T) { + s := newShape445() + s.c.State, s.c.Since = Delivered, at445.Add(9*time.Minute) + s.c.Walk.State = walkDone + for _, m := range modules445 { + s.c.Steps = append(s.c.Steps, Step{Machine: "novox", Module: m, State: StepPassed}, + Step{Machine: stepTheRest, Module: m, State: StepSent}) + } + s.a.ContainedIn = map[string]bool{mergedC: true} + s.walks = s.walks[:2] + store := newMemStore() + for _, d := range []*Delivery{s.a, s.b, s.c} { + if err := store.PutDelivery(d); err != nil { + t.Fatal(err) + } + } + g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", + Check: &GroupCheck{Verdict: "pass", Commits: commitsOf([]*Delivery{s.a, s.b})}} + if err := store.PutGroup(g); err != nil { + t.Fatal(err) + } + w := newWorldOver(t, store, newFakeController(), newFakeForge(), at445.Add(10*time.Minute)) + w.h.mu.Lock() + gg := w.h.groups[g445] + if w.h.deliveries[aID].CoveredBy != "" { + t.Fatal("A is marked covered already: the case is not the one under test") + } + w.h.turning = map[string]*turnWant{} + w.h.groupTurns(gg, w.h.membersOf(gg)) + b := w.h.deliveries[bID] + turned := w.h.turning["plan-lab"] != nil && containsString(w.h.turning["plan-lab"].ids, bID) + w.h.mu.Unlock() + if b.State != Published || !turned { + t.Fatalf("B is %s, at its turn %v; want its turn taken", b.State, turned) + } +} + +// foldShape is the review's shape: B and C on one walk, plan-2, not started, waiting for mesh-delivery's word. +func foldShape() *shape445 { + s := newShape445() + s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, + WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} + s.b.Walk = &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", + Branch: "main", Created: at445.Add(4 * time.Minute)} + two := ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", + Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}, {Repository: lab445, Commit: mergedB}}}) + two.Tiers = [][]string{modules445, {"mesh-lab"}} + two.Modules["mesh-lab"] = &WalkModule{} + s.walks = []Walk{s.walks[0], two} + return s +} + +// The turn on the carrier's walk is given only on the forge's word that the carrier contains A's merge: not yet +// said, B waits and the forge is asked; said "no", C is no carrier and B is stopped, never let go on plan-2. +func TestASharedTurnWaitsForTheForgesWord(t *testing.T) { + t.Run("not yet said: B waits and the forge is asked", func(t *testing.T) { + w := foldShape().open(t) + waits445(t, w, cID) + if containsString(w.ctl.delivers, "plan-2") { + t.Fatalf("plan-2 was let go %v before the forge said C contains A", w.ctl.delivers) + } + w.forge.mu.Lock() + asked := containsString(w.forge.containsAsks, mergedA+" in "+mergedC) + w.forge.mu.Unlock() + if !asked { + t.Fatal("the forge was not asked whether C's merge contains A's") + } + w.forge.contains[mergedA+" in "+mergedC] = true + w.later(2 * time.Minute) + w.settleAll() + letGo445(t, w, "plan-2") + }) + t.Run("said no: no shared turn", func(t *testing.T) { + s := foldShape() + s.forge = map[string]bool{mergedA + " in " + mergedC: false} + w := s.open(t) + if d := w.delivery(bID); d.State == Delivering || d.LetGo != (time.Time{}) { + t.Fatalf("B was given its turn on plan-2 (%s, let go %v)", d.State, d.LetGo) + } + stopped445(t, w, "does not contain its merge") + }) +} + +// Two carriers for A: C on B's own walk carries one module, D on a walk of its own the other. B is not given its +// turn on C's walk, which cannot deliver A whole: it waits for D's walk, goes once D covers its module, and is +// stopped, never delivering, when D's walk fails (re-review of #203). +func TestASharedTurnNeedsEveryModuleOnTheOneWalk(t *testing.T) { + two := func() *shape445 { + s := foldShape() + s.a.Plan = planMoving(ctl445, "mesh-controller", "route-proxy") + s.c.Plan = planMoving(ctl445, "mesh-controller") + s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", + Branch: "fix/other", State: Delivering, Since: at445.Add(5 * time.Minute), Created: at445.Add(-time.Hour), + MergedAs: mergedD, MergedAt: at445.Add(90 * time.Second), Plan: planMoving(ctl445, "route-proxy"), + Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-other", Commit: mergedD, State: walkRolling, + WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at445.Add(4 * time.Minute)}}} + went := at445.Add(5 * time.Minute) + other := ctlWalk("plan-other", mergedD, at445.Add(4*time.Minute), walkRolling, &WalkDelivery{ + Awaits: "mesh-delivery", Go: &went, By: byOwner, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}) + other.Modules = map[string]*WalkModule{"route-proxy": {}} + s.walks = append(s.walks, other) + s.forge = map[string]bool{mergedA + " in " + mergedC: true, mergedA + " in " + mergedD: true} + return s + } + t.Run("D's walk open: B waits for both", func(t *testing.T) { + w := two().open(t) + waits445(t, w, dID445) + if !strings.Contains(w.delivery(bID).TurnAwaits, cID) || containsString(w.ctl.delivers, "plan-2") { + t.Fatalf("B is %s, let go %v, waiting for %q", w.state(bID), w.ctl.delivers, w.delivery(bID).TurnAwaits) + } + }) + t.Run("D's walk delivered: B goes on C's walk", func(t *testing.T) { + w := two().open(t) + end445(w, "plan-other", walkDone) + letGo445(t, w, "plan-2") + }) + t.Run("D's walk failed: B is stopped, never delivering", func(t *testing.T) { + w := two().open(t) + end445(w, "plan-other", walkFailed) + d := w.delivery(bID) + for _, tr := range d.Transitions { + if tr.To == Delivering { + t.Fatalf("B was delivering before it was stopped: %+v", d.Transitions) + } + } + stopped445(t, w, "route-proxy") + }) +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 5f9964fb..25ec79e1 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -65,6 +65,11 @@ type Delivery struct { // (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316). CoveredBy string `json:"covered_by,omitempty"` Lifted string `json:"lifted,omitempty"` + // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, + // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, + // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. + TurnAwaits string `json:"turn_awaits,omitempty"` + TurnAwaitsUntil *time.Time `json:"turn_awaits_until,omitempty"` // ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's // merge (gitea_contains): asked once per pair, and kept (ADR 0270). ContainedIn map[string]bool `json:"contained_in,omitempty"` diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index ce8fa550..ad55d7ff 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -501,6 +501,13 @@ func boundOf(d *Delivery) (Bound, bool) { if d.waitsForItsOwnWalk() { return rewalkBound, true } + if d.State == Published && d.TurnAwaits != "" && d.TurnAwaitsUntil != nil { + // Its turn waits for a later merge to carry a member before it: bounded by that carrier's own bound, and + // said with it, never the generic "waits for its turn or its word" (novox/hq issue 445). + b := Bounds[Published] + b.For, b.Says = max(b.For, d.TurnAwaitsUntil.Sub(boundSince(d))), d.TurnAwaits + return b, true + } b, bounded := Bounds[d.State] return b, bounded }