diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 9223c804..8dcb0fc7 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -925,12 +925,16 @@ func (h *Holder) plan() []func() { // (novox/hq issue 319). A plan that moves nothing is delivered; one whose every module a later merge into the // same trunk delivered is covered. Read again each tick, so a delivery read back after a restart, or held // before this was known, ends by itself. + // A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what + // that walk did not build, a later delivered merge may have; and when the cover test says none did, past the + // grace, it is stopped saying so — never left waiting for a start that cannot come. for _, d := range h.deliveries { - if d.State != Held && !h.unwalkedPast(d, now) { + ended := d.endedUnstarted() + if d.State != Held && !h.unwalkedPast(d, now) && !ended { continue } + var ask []string if d.CoveredBy == "" { - var ask []string d.CoveredBy, ask = h.coverOf(d) for _, in := range ask { if a := h.askContains(d, in, now); a != nil { @@ -938,6 +942,11 @@ func (h *Holder) plan() []func() { } } } + if ended { + if a := h.strand(d, ask, now); a != nil { + asks = append(asks, a) + } + } if ts := settle(d, Facts{Now: now}); len(ts) > 0 { h.moved(d, ts) h.closeGroupOf(d) @@ -1002,6 +1011,40 @@ func (h *Holder) plan() []func() { return asks } +// strandedAfter is how long after its walk ended unstarted a delivery that walk did not deliver waits for a later +// merge to cover it before it is stopped (novox/hq issue 474): within the bound of a published delivery, so it is +// stopped, saying why, before `stalled` would list it. +const strandedAfter = 20 * time.Minute + +// strand says why a delivery whose walk ended done before its word is stopped, once that is so: the walk did not +// build every module it moves, the cover test asked the forge everything it had to and found no later delivered +// merge that carried them, and the walk ended strandedAfter ago. A walk record kept before it named its modules is +// asked for again (askWalk) rather than read as moving nothing. +func (h *Holder) strand(d *Delivery, pending []string, now time.Time) func() { + moved, known := d.walkMovedItsModules() + switch { + case !known && d.Plan != nil: + return h.askWalk(d.Walk.ID, now) + case !known, moved, d.CoveredBy != "", len(pending) > 0, d.Stranded != "": + return nil + } + ended := d.Walk.Updated + if ended.IsZero() { + ended = d.Since + } + if now.Sub(ended) < strandedAfter { + return nil + } + note := "" + if d.Walk.Note != "" { + note = " (" + d.Walk.Note + ")" + } + d.Stranded = fmt.Sprintf("its walk %s ended done before its word without building %s%s, and no later merge "+ + "into %s that contains it delivered them: nothing delivered it (novox/hq issue 474)", d.Walk.ID, + strings.Join(d.notMovedBy(), ", "), note, orNone(d.Base)) + return nil +} + // unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read // again after it: what holds it for a person, unless something already ends it. func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go new file mode 100644 index 00000000..bf65c1ab --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue474_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 474: a merge batched behind a later one of its branch (ADR 0276) whose walk ended done before +// any word was asked — the controller judged the batch's files moved nothing it holds, "nothing to walk" — was +// left published, waiting for "its walk's start", which a done walk never has. Its walk's record settles it: it +// is delivered when that walk moved its modules, covered when a later delivered merge carried them (ADR 0270), +// and otherwise stopped, saying why — never left waiting. +// +// The shape is the night's own (2026-10-11): mesh-controller #215 (merged as fbc7bc97, its check's plan moving +// build-agent) and #217 (merged as 8305e4e3 a minute later, moving nothing) in one batch, walked as +// plan-1791681624611167730 at 8305e4e3: done, no tier, still saying it awaits mesh-delivery. + +const ( + ctl474 = "novox/mesh-controller" + head215 = "d2e9dc61590f950b" + merge215 = "fbc7bc976b3c115b" + head217 = "68fbd99e894443ce" + merge217 = "8305e4e3d126594d" + head219 = "ef551fdfb6b29d4a" + merge219 = "0a2e58c0704ccc09" + batchWalk = "plan-1791681624611167730" + walk219 = "plan-1791686195314774150" + nothingNote = "moved nothing the mesh holds: nothing to walk" +) + +var at215 = time.Date(2026, 10, 11, 1, 20, 19, 0, time.UTC) + +// theNight is #215 and #217 checked and merged, and their batch's walk done with no tier, as the controller said it. +func theNight(t *testing.T) *world { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute)) + p215 := pr(ctl474, 215, head215, "fix/431-make-check-runs-what-the-gate-runs", "Makefile") + p217 := pr(ctl474, 217, head217, "fix/432-a-test-reads-what-it-carries", "cmd/mesh-controller/check_test.go") + w.h.PullUpdated(p215) + w.h.PullUpdated(p217) + w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent")) + w.h.Checked(verdictMoving(ctl474, 217, head217)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p215, merge215, at215)) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p217, merge217, at215.Add(56*time.Second))) + w.later(90 * time.Second) + cut := w.now + w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: merge217, Created: cut, + Updated: cut, State: walkDone, Tiers: [][]string{}, Modules: map[string]*WalkModule{}, Note: nothingNote, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: ctl474, Commit: merge215, Carried: merge217}, {Repository: ctl474, Commit: merge217}}}})) + w.settleAll() + return w +} + +// The live delivery: its walk ended without moving build-agent, and a later merge (#219) delivered build-agent +// everywhere from main holding its merge — covered, from the walk's record and the forge's word, never waiting. +func TestAMergeWhoseBatchWalkEndedIsCoveredByALaterDeliveredMerge(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + if s := w.state(IDOf(ctl474, head217)); s != Delivered { + t.Fatalf("#217, which moves nothing, is %s", s) + } + w.forge.contains[merge215+" in "+merge219] = true + deliverLater(w, ctl474, 219, "main", head219, merge219, at215.Add(76*time.Minute), walkDone, "build-agent") + w.later(2 * time.Minute) + w.settleAll() + d := w.delivery(id) + if d.State != Covered { + t.Fatalf("#215, its walk %s done with nothing moved and build-agent delivered later by #219, is %s, waiting "+ + "for %q", batchWalk, d.State, w.h.waitsFor(d)) + } + if why := d.Transitions[len(d.Transitions)-1].Why; !strings.Contains(why, IDOf(ctl474, head219)) { + t.Fatalf("covered, but not by #219: %q", why) + } + if len(w.h.Stalled()) != 0 { + t.Fatalf("stalled still lists %+v", w.h.Stalled()) + } +} + +// With nothing later to carry its modules, it is stopped once the cover test says so and the walk ended past the +// grace — saying which walk ended without moving what — and not before: a later merge may still cover it. +func TestAMergeWhoseBatchWalkEndedAndNothingCoversItIsStoppedSayingWhy(t *testing.T) { + w := theNight(t) + id := IDOf(ctl474, head215) + w.later(5 * time.Minute) + w.settleAll() + if s := w.state(id); s != Published { + t.Fatalf("within the grace it is %s", s) + } + w.later(strandedAfter) + w.settleAll() + d := w.delivery(id) + if d.State != Stopped { + t.Fatalf("its walk done without build-agent and nothing later carrying it, it is %s, waiting for %q", d.State, + w.h.waitsFor(d)) + } + why := d.Transitions[len(d.Transitions)-1].Why + for _, want := range []string{batchWalk, "build-agent", nothingNote} { + if !strings.Contains(why, want) { + t.Fatalf("its stop says %q, not %q", why, want) + } + } + if len(w.h.Stalled()) != 0 { + t.Fatalf("stalled still lists %+v", w.h.Stalled()) + } +} + +// A walk that ended done before a word, having built every module the merge moves, delivered it: a merge it +// carried for a later one is covered by that one, its own merge delivered. +func TestAMergeWhoseWalkEndedHavingMovedItsModulesIsSettledByIt(t *testing.T) { + for _, own := range []bool{true, false} { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), at215.Add(-time.Minute)) + p := pr(ctl474, 215, head215, "fix/x", "cmd/mesh-builder/main.go") + w.h.PullUpdated(p) + w.h.Checked(verdictMoving(ctl474, 215, head215, "build-agent")) + w.later(time.Minute) + w.h.PullMerged(mergedAt(p, merge215, at215)) + commit, merges := merge215, []WalkMerge{{Repository: ctl474, Commit: merge215}} + if !own { + commit, merges = merge217, []WalkMerge{{Repository: ctl474, Commit: merge215, Carried: merge217}} + } + w.later(2 * time.Minute) + sent := w.now + w.h.WalkMoved(w.ctl.put(Walk{ID: batchWalk, Repository: ctl474, Branch: "main", Commit: commit, Created: w.now, + Updated: w.now, State: walkDone, Tiers: [][]string{{"build-agent"}}, + Modules: map[string]*WalkModule{"build-agent": {State: "built", SentAt: &sent}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: merges}})) + w.settleAll() + want := Delivered + if !own { + want = Covered + } + if d := w.delivery(IDOf(ctl474, head215)); d.State != want { + t.Fatalf("own commit %v: it is %s, waiting for %q, not %s", own, d.State, w.h.waitsFor(d), want) + } + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 25ec79e1..0e92e6e5 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -65,6 +65,9 @@ type Delivery struct { // (novox/hq issue 319); Lifted why what held it for a person no longer holds (issue 316). CoveredBy string `json:"covered_by,omitempty"` Lifted string `json:"lifted,omitempty"` + // Stranded says why a merge whose walk ended done before its word, without moving every module it moves, is + // stopped: nothing later delivered them either (novox/hq issue 474). Set once the cover test says no. + Stranded string `json:"stranded,omitempty"` // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. @@ -469,6 +472,11 @@ type WalkSeen struct { Created time.Time `json:"created"` // TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it. TakenOverBy string `json:"taken_over_by,omitempty"` + // Moved are the modules the walk builds, as its record names them; MovedSaid says they were read, which a + // record kept before novox/hq issue 474 did not. What a walk that ended before its word delivered is read + // from them. + Moved []string `json:"moved,omitempty"` + MovedSaid bool `json:"moved_said,omitempty"` } // Seen is a walk as a delivery keeps it. @@ -484,9 +492,53 @@ func (w Walk) Seen() *WalkSeen { s.Branch = w.Branch s.Created = w.Created s.TakenOverBy = w.takenOverBy() + for name := range w.Modules { + s.Moved = append(s.Moved, name) + } + sort.Strings(s.Moved) + s.MovedSaid = true return s } +// endedUnstarted is whether a published delivery's walk ended done without ever being let go: a batch's walk the +// controller judged to move nothing it holds is done when it is cut (novox/hq issue 474). No start can come for it, +// so its record settles the delivery. +func (d *Delivery) endedUnstarted() bool { + return d.State == Published && d.Walk != nil && d.Walk.State == walkDone && !d.Walk.Started() +} + +// walkMovedItsModules is whether the walk it follows built every module its plan moves, and whether that is known: +// from the modules the walk's record names, or, for a record kept before they were, from a walk of no tier, which +// moved nothing. Unknown when its plan is not known. +func (d *Delivery) walkMovedItsModules() (moved, known bool) { + if d.Walk == nil || d.Plan == nil { + return false, false + } + if !d.Walk.MovedSaid && d.Walk.Tiers > 0 { + return false, false + } + for _, m := range d.Plan.Moved { + if !containsString(d.Walk.Moved, m) { + return false, true + } + } + return true, true +} + +// notMovedBy is the modules its plan moves that the walk it follows did not build. +func (d *Delivery) notMovedBy() []string { + var out []string + if d.Walk == nil || d.Plan == nil { + return nil + } + for _, m := range d.Plan.Moved { + if !containsString(d.Walk.Moved, m) { + out = append(out, m) + } + } + return out +} + // Started is whether the walk is past its wait: it waited for nobody, or was let go — whatever came of it // since. A walk superseded or stopped while it waited never started. func (s *WalkSeen) Started() bool { return s != nil && !s.Waits && (s.WaitedFor == "" || s.GoBy != "") } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index ad55d7ff..76005611 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -231,8 +231,21 @@ var Table = []Row{ return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded") }}, {From: []State{Published}, Event: EvDone, To: Delivered, - Guard: "nothing for a walk to move", - Holds: func(d *Delivery, f Facts) error { return unless(d.Walk == nil && d.MovesNothing(), "a walk moves it") }}, + Guard: "nothing for a walk to move; or its walk, carrying its own commit, ended done before its word having built " + + "every module it moves (novox/hq issue 474)", + Holds: func(d *Delivery, f Facts) error { + if d.Walk == nil && d.MovesNothing() { + return nil + } + moved, _ := d.walkMovedItsModules() + return unless(d.endedUnstarted() && !d.carriedLater() && moved, "a walk moves it") + }, + Says: func(d *Delivery) string { + if d.Walk == nil { + return "nothing for a walk to move" + } + return fmt.Sprintf("its walk %s ended done before its word, having built every module it moves", d.Walk.ID) + }}, {From: []State{Published, Ready}, Event: EvHold, To: Held, Guard: "something holds it for a person: its group's composed check did not pass for the heads that merged, " + "or it merged and no walk was opened for it", @@ -248,6 +261,34 @@ var Table = []Row{ } return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") }}, + // **A walk that ended before its word settles its delivery** (novox/hq issue 474): a batch's walk the + // controller judged to move nothing it holds is done when it is cut, and a merge it carried was left published, + // waiting for a start that cannot come. Covered when that walk, carrying a later merge for it, built its modules, + // or when a later delivered merge carried them (ADR 0270's covered test); stopped, saying why, when neither. + {From: []State{Published}, Event: EvCovered, To: Covered, + Guard: "its walk ended done before its word, and its modules were delivered by a later merge that contains it: " + + "that walk, carrying it, built them, or a later merge's finished walk sent them everywhere (ADR 0270)", + Holds: func(d *Delivery, f Facts) error { + if !d.endedUnstarted() { + return errors.New("its walk has not ended before its word") + } + moved, _ := d.walkMovedItsModules() + return unless(d.carriedLater() && moved || d.MergedAs != "" && d.CoveredBy != "", + "nothing later delivered its modules") + }, + Says: func(d *Delivery) string { + if moved, _ := d.walkMovedItsModules(); d.carriedLater() && moved { + return "covered: a later merge that contains it was delivered: " + d.Carrier + } + return "covered: a later merge that contains it was delivered: " + d.CoveredBy + }}, + {From: []State{Published}, Event: EvStop, To: Stopped, + Guard: "its walk ended done before its word without building every module it moves, and no later merge that " + + "contains it delivered them: no start can come, and nothing delivered it (novox/hq issue 474)", + Holds: func(d *Delivery, f Facts) error { + return unless(d.endedUnstarted() && d.Stranded != "", "its walk has not ended unstarted, or something may cover it") + }, + Says: func(d *Delivery) string { return d.Stranded }}, {From: []State{Held}, Event: EvGo, To: Delivering, Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " + "nobody, or a person's `plans go`", diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index 072def6c..75f5fecc 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -48,6 +48,11 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { if r.Event == EvStop && !r.Act { f = Facts{Now: now} d.Walk = &WalkSeen{ID: "plan-1", State: walkFailed, StoppedBy: "mesh-delivery for jochen"} + if r.From[0] == Published { + // Its walk ended done before its word, and nothing delivered it (novox/hq issue 474). + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone, Waits: true, WaitedFor: "mesh-delivery", MovedSaid: true} + d.Stranded = "its walk plan-1 ended done before its word without building app" + } } case EvReask: f.Why, f.By = "delivery stalled", byH2 @@ -65,6 +70,10 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { } case EvCovered: d.MergedAs, d.CoveredBy = "aaaa1111", "delivered with novox/app@bbbb (merged as bbbb2222, walk plan-2): app" + if r.From[0] == Published { + // Its walk ended done before its word; a later merge delivered its modules (novox/hq issue 474). + d.Walk = &WalkSeen{ID: "plan-1", State: walkDone, Waits: true, WaitedFor: "mesh-delivery", MovedSaid: true} + } if r.From[0] == Delivering { // Answered by its successor in one walk (ADR 0276): the walk done, carrying a later merge. d.CoveredBy, d.AnsweredBy, d.Carrier = "", "bbbb2222", "novox/app@bbbb (merged as bbbb2222, walk plan-2)" diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index e1045ac6..b9f0f2c1 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -50,6 +50,11 @@ func (h *Holder) waitsFor(d *Delivery) string { if d.Walk != nil && d.Walk.TakenOverBy != "" { return "the walk " + d.Walk.TakenOverBy + ", which took its walk " + d.Walk.ID + " over" } + if d.endedUnstarted() { + // Never "its walk's start": a done walk has none (novox/hq issue 474). + return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": its walk " + d.Walk.ID + + " ended without building it, and it is stopped " + strandedAfter.String() + " after that walk ended otherwise" + } if d.Group != "" { return "its turn in group " + d.Group }