package main import ( "strings" "testing" "time" ) // novox/hq issue 445: in a delivery group ordered [A, B], A's walk was superseded by the walk of a later merge C // of A's repository that does not answer A's merge. groupTurns stopped B at once, although C's walk could still // carry A's modules and A then counts as reached (ADR 0270 decision 2, issue 442). While a later merge that can // carry A is open, B waits, saying for which merge and until when; once that merge's walk is delivered and covers // A, B takes its turn; when C's walk is B's own (ADR 0276 folds both into one successor), B takes its turn on it; // when no later merge can carry A, or the one that could is past its own bound, B is stopped, saying why. // // The shape is the live group fix/assign-says-why-a-module-is-not-there as mesh-delivery showed it on // 2026-10-11 (issue 442's test): the controller's member superseded first in the order, the lab's after it. const ( g445 = "fix/assign-says-why-a-module-is-not-there" ctl445, lab445 = "novox/mesh-controller", "novox/mesh-lab" mergedA, mergedB, mergedC = "a5f53ef9ebdf3650", "77777777aaaaaaaa", "3f68a49500000000" mergedD = "4d4d4d4d00000000" headA, headB, headC, headD = "8adb7f1a051941e9", "33e63ee5b47a7552", "3f68a495c0c0c0c0", "4d4d4d4dc0c0c0c0" ) var ( at445 = time.Date(2026, 10, 7, 19, 4, 10, 0, time.UTC) modules445 = []string{"build-agent", "mesh-controller", "route-proxy"} aID, bID, cID, dID445 = IDOf(ctl445, headA), IDOf(lab445, headB), IDOf(ctl445, headC), IDOf(ctl445, headD) ) // shape445 is A superseded, B published on its walk, and C, the later merge of A's repository, as each case sets it. type shape445 struct { a, b, c *Delivery extra []*Delivery walks []Walk forge map[string]bool // what the forge says of containment from the start } func newShape445() *shape445 { at := at445 s := &shape445{} s.a = &Delivery{ID: aID, Repository: ctl445, Commit: headA, Number: 109, Base: "main", Branch: g445, Group: g445, State: Superseded, Since: at.Add(4 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedA, MergedAt: at, Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-a5f53e", Commit: mergedA, State: walkSuperseded, Branch: "main", Note: "superseded at tier 0 by plan-later (" + ctl445 + " at 3f68a495); planned there again"}, Transitions: []Transition{{At: at.Add(4 * time.Minute), From: Published, To: Superseded, Event: EvSuperseded, Why: "a newer delivery to the same trunk took over its walk"}}} s.b = &Delivery{ID: bID, Repository: lab445, Commit: headB, Number: 117, Base: "main", Branch: g445, Group: g445, State: Published, Since: at.Add(3 * time.Minute), Created: at.Add(-2 * time.Hour), MergedAs: mergedB, MergedAt: at.Add(2 * time.Minute), Plan: planMoving(lab445, "mesh-lab"), Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-lab", Commit: mergedB, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at.Add(3 * time.Minute)}} s.c = &Delivery{ID: cID, Repository: ctl445, Commit: headC, Number: 110, Base: "main", Branch: "fix/later", State: Delivering, Since: at.Add(5 * time.Minute), Created: at.Add(-time.Hour), MergedAs: mergedC, MergedAt: at.Add(time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-later", Commit: mergedC, State: walkRolling, WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at.Add(4 * time.Minute)}} went := at.Add(5 * time.Minute) s.walks = []Walk{ {ID: "plan-a5f53e", Repository: ctl445, Branch: "main", Commit: mergedA, Created: at.Add(30 * time.Second), State: walkSuperseded, Note: s.a.Walk.Note, Modules: map[string]*WalkModule{}}, {ID: "plan-lab", Repository: lab445, Branch: "main", Commit: mergedB, Created: at.Add(3 * time.Minute), State: walkBuilding, Tiers: [][]string{{"mesh-lab"}}, Modules: map[string]*WalkModule{"mesh-lab": {}}, Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}, ctlWalk("plan-later", mergedC, at.Add(4*time.Minute), walkRolling, &WalkDelivery{Awaits: "mesh-delivery", Go: &went, By: byOwner}), } return s } // ctlWalk is a walk of the controller's repository at a merge commit, moving A's modules. func ctlWalk(id, commit string, created time.Time, state string, wd *WalkDelivery) Walk { w := Walk{ID: id, Repository: ctl445, Branch: "main", Commit: commit, Created: created, State: state, Tiers: [][]string{modules445}, Modules: map[string]*WalkModule{}, Delivery: wd} for _, m := range modules445 { w.Modules[m] = &WalkModule{} } return w } // open is the world over the shape, read back and ticked ten minutes after A's merge. func (s *shape445) open(t *testing.T) *world { t.Helper() store := newMemStore() for _, d := range append([]*Delivery{s.a, s.b, s.c}, s.extra...) { if d == nil { continue } if err := store.PutDelivery(d); err != nil { t.Fatal(err) } } g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", OrderOf: map[string]string{}, Check: &GroupCheck{Asked: "check-1", At: at445.Add(-time.Hour), Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{s.a, s.b})}} if err := store.PutGroup(g); err != nil { t.Fatal(err) } ctlr := newFakeController() for _, w := range s.walks { ctlr.put(w) } forge := newFakeForge() for k, v := range s.forge { forge.contains[k] = v } w := newWorldOver(t, store, ctlr, forge, at445.Add(10*time.Minute)) if err := w.h.Reconcile(); err != nil { t.Fatal(err) } w.settleAll() return w } // end ends a walk in the state given, every module sent to every machine running it. func end445(w *world, walk, state string) { r := w.ctl.walk(walk) sent := w.now for m := range r.Modules { r.Modules[m] = &WalkModule{State: "built", SentAt: &sent} } r.State = state w.h.WalkMoved(w.ctl.put(r)) w.settleAll() } func waits445(t *testing.T, w *world, on string) { t.Helper() d := w.delivery(bID) if d.State != Published || containsString(w.ctl.delivers, "plan-lab") { t.Fatalf("B is %s, let go %v; want it published and waiting while %s can still carry A", d.State, w.ctl.delivers, on) } if !strings.Contains(d.TurnAwaits, on) || !strings.Contains(d.TurnAwaits, aID) || d.TurnAwaitsUntil == nil { t.Fatalf("B says it waits for %q until %v; want it to name A and %s", d.TurnAwaits, d.TurnAwaitsUntil, on) } g := w.h.groups[g445] if state, why := GroupState(g, w.h.membersOf(g)); state != "delivering" { t.Fatalf("the group reads %s (%s), want delivering", state, why) } } func stopped445(t *testing.T, w *world, says string) { t.Helper() d := w.delivery(bID) if d.State != Stopped || len(d.Transitions) == 0 { t.Fatalf("B is %s, want stopped", d.State) } why := d.Transitions[len(d.Transitions)-1].Why if !strings.Contains(why, aID) || !strings.Contains(why, says) { t.Fatalf("B was stopped saying %q, want it to name %s and say %q", why, aID, says) } if containsString(w.ctl.delivers, "plan-lab") || d.TurnAwaits != "" { t.Fatalf("B's walk was let go %v, or B still says it waits: %q", w.ctl.delivers, d.TurnAwaits) } } func letGo445(t *testing.T, w *world, walk string) { t.Helper() if d := w.delivery(bID); d.State != Delivering || !containsString(w.ctl.delivers, walk) || d.TurnAwaits != "" { t.Fatalf("B is %s, let go %v, waiting for %q; want its turn taken on %s", d.State, w.ctl.delivers, d.TurnAwaits, walk) } } // C's walk runs on its own: B waits for it, says so, and goes on or stops by what comes of it. func TestAMemberAfterACarriableSupersededOneWaitsForItsCarrier(t *testing.T) { t.Run("a later walk that can carry A is open: B waits, naming it", func(t *testing.T) { w := newShape445().open(t) waits445(t, w, cID) // Past a published delivery's own 30 minutes, B is not stalled for "its turn or its word": its bound is // C's, and what `stalled` would say names C. w.later(Bounds[Published].For + 5*time.Minute) w.settleAll() waits445(t, w, cID) for _, l := range w.h.Stalled() { if l.ID == bID { t.Fatalf("B is listed stalled while C's walk is within its bound: %+v", l) } } if b, _ := boundOf(w.delivery(bID)); !strings.Contains(b.Says, cID) || !strings.Contains(b.Says, "until") { t.Fatalf("B's bound says %q", b.Says) } }) t.Run("that walk delivered and covers A: B takes its turn", func(t *testing.T) { w := newShape445().open(t) w.forge.contains[mergedA+" in "+mergedC] = true end445(w, "plan-later", walkDone) if s := w.state(cID); s != Delivered { t.Fatalf("C is %s", s) } if d := w.delivery(aID); d.State != Superseded || d.CoveredBy == "" { t.Fatalf("A is %s, covered by %q", d.State, d.CoveredBy) } letGo445(t, w, "plan-lab") }) t.Run("the forge says the later merge does not contain A: B is stopped", func(t *testing.T) { w := newShape445().open(t) w.forge.contains[mergedA+" in "+mergedC] = false end445(w, "plan-later", walkDone) stopped445(t, w, "can carry") }) t.Run("the later walk failed: B is stopped", func(t *testing.T) { w := newShape445().open(t) end445(w, "plan-later", walkFailed) if s := w.state(cID); s != Failed { t.Fatalf("C is %s", s) } stopped445(t, w, "can carry") }) t.Run("the later walk stays open past its own bound: B is stopped", func(t *testing.T) { w := newShape445().open(t) w.later(Bounds[Delivering].For - 10*time.Minute) w.settleAll() waits445(t, w, cID) w.later(10 * time.Minute) w.settleAll() stopped445(t, w, "past its bound") }) t.Run("the forge never says whether the delivered walk contains A: B is stopped after the grace", func(t *testing.T) { w := newShape445().open(t) end445(w, "plan-later", walkDone) waits445(t, w, cID) w.later(containsGrace + time.Minute) w.settleAll() stopped445(t, w, "did not say") }) } // The review's shape of #203 (ADR 0276): A's waiting walk and B's were folded into one successor, plan-2, that // answers C's merge and B's and not A's. The carrier is in B's own walk, which waits for B's turn: B takes its // turn there, and the one walk delivers C and B; A is covered once it is done. func TestAMemberWhoseWalkCarriesTheSupersededOneBeforeItTakesItsTurn(t *testing.T) { s := foldShape() s.forge = map[string]bool{mergedA + " in " + mergedC: true} w := s.open(t) letGo445(t, w, "plan-2") if s := w.state(cID); s != Delivering { t.Fatalf("C is %s", s) } end445(w, "plan-2", walkDone) g := w.h.groups[g445] if state, why := GroupState(g, w.h.membersOf(g)); state != "delivered" { t.Fatalf("the group reads %s (%s), want delivered", state, why) } } // A carrier superseded in turn hands the wait to the later merge that took it over: C's waiting walk is folded // into D's, which answers only D; B waits for D, and goes once D's walk covers A. func TestTheWaitFollowsACarrierSupersededInTurn(t *testing.T) { s := newShape445() s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} s.walks[2] = ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}) s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", Branch: "fix/later-still", State: Ready, Since: at445.Add(6 * time.Minute), Created: at445.Add(-time.Hour), MergedAs: mergedD, MergedAt: at445.Add(6 * time.Minute), Plan: planMoving(ctl445, modules445...), Check: &Verdict{Gate: "pass"}}} w := s.open(t) // C is a lone delivery at its turn; the fake controller lets plan-2 go. It is folded all the same, as the // review's shape has it, before it started. two := w.ctl.walk("plan-2") two.Delivery.Go = nil two.State, two.Note = walkSuperseded, "superseded at tier 0 by plan-3 ("+ctl445+" at 4d4d4d4d); planned there again" w.h.WalkMoved(w.ctl.put(ctlWalk("plan-3", mergedD, at445.Add(7*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}))) w.h.WalkMoved(w.ctl.put(two)) w.settleAll() if s := w.state(cID); s != Superseded { t.Fatalf("C is %s", s) } waits445(t, w, dID445) if strings.Contains(w.delivery(bID).TurnAwaits, cID) { t.Fatalf("B still waits for C: %q", w.delivery(bID).TurnAwaits) } w.forge.contains[mergedA+" in "+mergedD] = true end445(w, "plan-3", walkDone) w.later(2 * time.Minute) // the forge, asked while it could not say, is asked again a minute on w.settleAll() if d := w.delivery(aID); d.CoveredBy == "" || !strings.Contains(d.CoveredBy, dID445) { t.Fatalf("A is covered by %q", d.CoveredBy) } letGo445(t, w, "plan-lab") } // A merged carrier still in an assembling batch has no state bound of its own: it is awaited for // readyCarrierBound from its merge, and then B is stopped. func TestACarrierInABatchIsAwaitedWithinTheWindowAndOneWalk(t *testing.T) { s := newShape445() s.c.State, s.c.Walk = Ready, nil closes := at445.Add(3 * time.Minute) s.walks[2] = Walk{ID: "plan-batch", Repository: ctl445, Branch: "main", Commit: mergedC, State: walkQueued, Created: at445.Add(2 * time.Minute), Delivery: &WalkDelivery{Awaits: "mesh-delivery", Batch: &WalkBatch{ ClosesAt: &closes, Behind: "plan-other"}, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}}}} w := s.open(t) waits445(t, w, cID) // The merge window at its longest, then the one open walk the batch may queue behind (ADR 0276 decisions 1, // 4 and 9). if readyCarrierBound != 10*time.Minute+Bounds[Delivering].For { t.Fatalf("a ready carrier is awaited %s, want the window's 10 minutes and one walk's %s", readyCarrierBound, Bounds[Delivering].For) } until := s.c.MergedAt.Add(readyCarrierBound) if got := w.delivery(bID).TurnAwaitsUntil; got == nil || !got.Equal(until) { t.Fatalf("B waits until %v, want %v", got, until) } // Within the bound, past every state bound a ready delivery could be read against, B still waits. w.later(until.Add(-time.Minute).Sub(w.now)) w.settleAll() waits445(t, w, cID) w.later(2 * time.Minute) w.settleAll() stopped445(t, w, "no walk carried it within "+readyCarrierBound.String()) } // A member whose every module a later delivered merge carried, the forge saying it contains the member's merge, // is reached even before CoveredBy is written on it: never a stop with an empty reason. func TestACarriedMemberNotYetMarkedCoveredIsReached(t *testing.T) { s := newShape445() s.c.State, s.c.Since = Delivered, at445.Add(9*time.Minute) s.c.Walk.State = walkDone for _, m := range modules445 { s.c.Steps = append(s.c.Steps, Step{Machine: "novox", Module: m, State: StepPassed}, Step{Machine: stepTheRest, Module: m, State: StepSent}) } s.a.ContainedIn = map[string]bool{mergedC: true} s.walks = s.walks[:2] store := newMemStore() for _, d := range []*Delivery{s.a, s.b, s.c} { if err := store.PutDelivery(d); err != nil { t.Fatal(err) } } g := &Group{ID: g445, Members: []string{aID, bID}, Closed: true, Order: []string{aID, bID}, Said: "delivering", Check: &GroupCheck{Verdict: "pass", Commits: commitsOf([]*Delivery{s.a, s.b})}} if err := store.PutGroup(g); err != nil { t.Fatal(err) } w := newWorldOver(t, store, newFakeController(), newFakeForge(), at445.Add(10*time.Minute)) w.h.mu.Lock() gg := w.h.groups[g445] if w.h.deliveries[aID].CoveredBy != "" { t.Fatal("A is marked covered already: the case is not the one under test") } w.h.turning = map[string]*turnWant{} w.h.groupTurns(gg, w.h.membersOf(gg)) b := w.h.deliveries[bID] turned := w.h.turning["plan-lab"] != nil && containsString(w.h.turning["plan-lab"].ids, bID) w.h.mu.Unlock() if b.State != Published || !turned { t.Fatalf("B is %s, at its turn %v; want its turn taken", b.State, turned) } } // foldShape is the review's shape: B and C on one walk, plan-2, not started, waiting for mesh-delivery's word. func foldShape() *shape445 { s := newShape445() s.c.State, s.c.Walk = Published, &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} s.b.Walk = &WalkSeen{ID: "plan-2", Commit: mergedC, State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery", Branch: "main", Created: at445.Add(4 * time.Minute)} two := ctlWalk("plan-2", mergedC, at445.Add(4*time.Minute), walkBuilding, &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: ctl445, Commit: mergedC}, {Repository: lab445, Commit: mergedB}}}) two.Tiers = [][]string{modules445, {"mesh-lab"}} two.Modules["mesh-lab"] = &WalkModule{} s.walks = []Walk{s.walks[0], two} return s } // The turn on the carrier's walk is given only on the forge's word that the carrier contains A's merge: not yet // said, B waits and the forge is asked; said "no", C is no carrier and B is stopped, never let go on plan-2. func TestASharedTurnWaitsForTheForgesWord(t *testing.T) { t.Run("not yet said: B waits and the forge is asked", func(t *testing.T) { w := foldShape().open(t) waits445(t, w, cID) if containsString(w.ctl.delivers, "plan-2") { t.Fatalf("plan-2 was let go %v before the forge said C contains A", w.ctl.delivers) } w.forge.mu.Lock() asked := containsString(w.forge.containsAsks, mergedA+" in "+mergedC) w.forge.mu.Unlock() if !asked { t.Fatal("the forge was not asked whether C's merge contains A's") } w.forge.contains[mergedA+" in "+mergedC] = true w.later(2 * time.Minute) w.settleAll() letGo445(t, w, "plan-2") }) t.Run("said no: no shared turn", func(t *testing.T) { s := foldShape() s.forge = map[string]bool{mergedA + " in " + mergedC: false} w := s.open(t) if d := w.delivery(bID); d.State == Delivering || d.LetGo != (time.Time{}) { t.Fatalf("B was given its turn on plan-2 (%s, let go %v)", d.State, d.LetGo) } stopped445(t, w, "does not contain its merge") }) } // Two carriers for A: C on B's own walk carries one module, D on a walk of its own the other. B is not given its // turn on C's walk, which cannot deliver A whole: it waits for D's walk, goes once D covers its module, and is // stopped, never delivering, when D's walk fails (re-review of #203). func TestASharedTurnNeedsEveryModuleOnTheOneWalk(t *testing.T) { two := func() *shape445 { s := foldShape() s.a.Plan = planMoving(ctl445, "mesh-controller", "route-proxy") s.c.Plan = planMoving(ctl445, "mesh-controller") s.extra = []*Delivery{{ID: dID445, Repository: ctl445, Commit: headD, Number: 111, Base: "main", Branch: "fix/other", State: Delivering, Since: at445.Add(5 * time.Minute), Created: at445.Add(-time.Hour), MergedAs: mergedD, MergedAt: at445.Add(90 * time.Second), Plan: planMoving(ctl445, "route-proxy"), Check: &Verdict{Gate: "pass"}, Walk: &WalkSeen{ID: "plan-other", Commit: mergedD, State: walkRolling, WaitedFor: "mesh-delivery", GoBy: byOwner, Branch: "main", Created: at445.Add(4 * time.Minute)}}} went := at445.Add(5 * time.Minute) other := ctlWalk("plan-other", mergedD, at445.Add(4*time.Minute), walkRolling, &WalkDelivery{ Awaits: "mesh-delivery", Go: &went, By: byOwner, Merges: []WalkMerge{{Repository: ctl445, Commit: mergedD}}}) other.Modules = map[string]*WalkModule{"route-proxy": {}} s.walks = append(s.walks, other) s.forge = map[string]bool{mergedA + " in " + mergedC: true, mergedA + " in " + mergedD: true} return s } t.Run("D's walk open: B waits for both", func(t *testing.T) { w := two().open(t) waits445(t, w, dID445) if !strings.Contains(w.delivery(bID).TurnAwaits, cID) || containsString(w.ctl.delivers, "plan-2") { t.Fatalf("B is %s, let go %v, waiting for %q", w.state(bID), w.ctl.delivers, w.delivery(bID).TurnAwaits) } }) t.Run("D's walk delivered: B goes on C's walk", func(t *testing.T) { w := two().open(t) end445(w, "plan-other", walkDone) letGo445(t, w, "plan-2") }) t.Run("D's walk failed: B is stopped, never delivering", func(t *testing.T) { w := two().open(t) end445(w, "plan-other", walkFailed) d := w.delivery(bID) for _, tr := range d.Transitions { if tr.To == Delivering { t.Fatalf("B was delivering before it was stopped: %+v", d.Transitions) } } stopped445(t, w, "route-proxy") }) }