diff --git a/.gitignore b/.gitignore index cc53768f..dfd78dd4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,9 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools modules/messenger/cmd/messenger/messenger modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher modules/claude-code/cmd/claude-code/claude-code +modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery # ...and the same built from the module's root (go build ./cmd/), which names it after the module. modules/messenger/messenger modules/claude-code/claude-code modules/mesh-watcher/mesh-watcher +modules/mesh-delivery/mesh-delivery diff --git a/modules/mesh-delivery/cmd/mesh-delivery/answers.go b/modules/mesh-delivery/cmd/mesh-delivery/answers.go new file mode 100644 index 00000000..13467d97 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/answers.go @@ -0,0 +1,414 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Which merges a walk answers (novox/hq issue 362, ADR 0276). +// +// On 2026-10-10 two merges of the catalogue 18 seconds apart were walked by one walk at the later merge's +// commit, which superseded the first. This owner found the walk a delivery follows by the walk's id or by the +// commit it was merged as, final or not: the surviving walk matched the dunst delivery, which the supersession +// had already ended, so nothing gave it its word, and the claude-code delivery, merged as another commit, +// matched no walk and was held for a person after ten minutes, for a reason that named neither. +// +// A walk answers every merge it carries: the merges it names (`delivery.merges`, said by a controller that +// batches merges), the merge of its own commit, a merge whose walk it took over, and a merge of its repository +// and branch the forge says its commit contains. A delivery follows the newest walk that answers it, and only a +// delivery not yet ended is ever tied to a walk by what it answers: an ended delivery is moved only by the +// record of its own walk, which no longer moves it. + +// seenWalk is a walk's last record and when it was heard. +type seenWalk struct { + walk Walk + at time.Time +} + +// keptWalkFor is how long a walk no longer open is kept for a merge heard after it ended; a batch is kept as +// long, past its window's latest close. +const keptWalkFor = 10 * time.Minute + +// answer is a delivery a walk answers and the commit of its repository the walk carries for it. +type answer struct { + d *Delivery + carried string +} + +// walkAnswers is whether a walk answers a delivery's merge, and the commit it carries for it. +func walkAnswers(w Walk, d *Delivery) (string, bool) { + if w.Release != nil || w.IsBatch() || d.MergedAs == "" { + return "", false + } + if c, ok := w.carriedFor(d.Repository, d.MergedAs); ok { + return c, true + } + if d.Walk != nil && d.Walk.TakenOverBy == w.ID && namesNoMerges(w) && strings.EqualFold(w.Repository, d.Repository) && + sameBranch(w.Branch, d.Base) { + // The walk that took its walk over, from a controller that names no merges: it answers every merge the one + // it took over answered, by its commit of the same repository and branch. A walk that names its merges + // answers only those (carriedFor, above). + return w.Commit, true + } + return "", false +} + +// containedAnswer is, for a controller that does not name the merges a walk answers, whether an open walk +// answers a ready merged delivery by containment: of its repository and branch, made for a merge after it, the +// forge says its commit contains this merge, and it moves every module this merge's plan moves. A later walk that +// moves only what its own merge moved does not deliver this one, however much of the trunk it contains. +func (h *Holder) containedAnswer(w Walk, d *Delivery) (string, bool) { + if !h.mayContain(w, d) || !d.ContainedIn[w.Commit] { + return "", false + } + return w.Commit, true +} + +// mayContain is whether an open walk is one to ask the forge about for a ready merged delivery (containedAnswer). +func (h *Holder) mayContain(w Walk, d *Delivery) bool { + if w.Release != nil || w.IsBatch() || !w.Open() || !namesNoMerges(w) || d.State != Ready || d.MergedAs == "" || d.Walk != nil || + d.Plan == nil || len(d.Plan.Moved) == 0 || !strings.EqualFold(w.Repository, d.Repository) || + !sameBranch(w.Branch, d.Base) || sameCommit(w.Commit, d.MergedAs) { + return false + } + for _, m := range d.Plan.Moved { + if _, moves := w.Modules[m]; !moves { + return false + } + } + // Made for a merge after this one: the walk's own merge, where this owner holds it, or else the walk itself. + after := w.Created + for _, o := range h.deliveries { + if strings.EqualFold(o.Repository, w.Repository) && sameCommit(o.MergedAs, w.Commit) && !o.MergedAt.IsZero() { + after = o.MergedAt + } + } + return after.After(d.MergedAt) +} + +// namesNoMerges is whether a walk is from a controller that does not name the merges it answers. +func namesNoMerges(w Walk) bool { return w.Delivery == nil || len(w.Delivery.Merges) == 0 } + +func sameBranch(a, b string) bool { return a == "" || b == "" || strings.EqualFold(a, b) } + +// follows is whether a delivery takes a walk that answers it as the one it follows: it has none, it is that walk, +// that walk took its own over, or that walk is newer than the one it follows. A walk record kept before its +// creation was read is followed past only once it is superseded or failed. +func follows(d *Delivery, w Walk) bool { + switch { + case d.Walk == nil || d.Walk.ID == w.ID || d.Walk.TakenOverBy == w.ID: + return true + case d.Walk.Created.IsZero(): + return d.Walk.State == walkSuperseded || d.Walk.State == walkFailed + } + return w.Created.After(d.Walk.Created) +} + +// deliveriesWalking are the deliveries a walk record moves: every delivery that already follows it — ended or +// not, its own record — and every delivery not ended that it answers and that follows it now. In id order. +func (h *Holder) deliveriesWalking(w Walk) []answer { + var out []answer + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + carried := d.AnsweredBy + if c, ok := walkAnswers(w, d); ok { + carried = c + } + out = append(out, answer{d: d, carried: carried}) + continue + } + if d.State.Final() || !follows(d, w) { + continue + } + if c, ok := walkAnswers(w, d); ok { + out = append(out, answer{d: d, carried: c}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].d.ID < out[j].d.ID }) + return out +} + +// accounted is whether any delivery, ended or not, follows the walk or is answered by it: a walk known for +// what it answers is never made a delivery of its own. +func (h *Holder) accounted(w Walk) bool { + for _, d := range h.deliveries { + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. +func (h *Holder) heldWalk(w Walk) bool { + for _, d := range h.deliveries { + if d.State.Final() { + continue + } + if d.Walk != nil && d.Walk.ID == w.ID { + return true + } + if _, ok := walkAnswers(w, d); ok { + return true + } + } + return false +} + +// pickWalk is the newest walk heard that answers a merged delivery with no walk yet; nil for none. +func (h *Holder) pickWalk(d *Delivery) (Walk, string, bool) { + var best Walk + var carried string + found := false + for _, s := range h.walks { + if !follows(d, s.walk) { + continue + } + c, ok := walkAnswers(s.walk, d) + if !ok { + c, ok = h.containedAnswer(s.walk, d) + } + if !ok { + continue + } + if !found || s.walk.Created.After(best.Created) || s.walk.Created.Equal(best.Created) && s.walk.ID > best.ID { + best, carried, found = s.walk, c, true + } + } + return best, carried, found +} + +// linkAnswering ties a merged delivery with no walk to the newest walk heard that answers it; under the lock. +func (h *Holder) linkAnswering(d *Delivery) bool { + if d.State.Final() || d.MergedAs == "" || d.Walk != nil { + return false + } + w, carried, ok := h.pickWalk(d) + if !ok { + return false + } + h.linkWalk(d, w, carried) + delete(h.unmatched, w.ID) + return true +} + +// containsAsks are the forge's asks whether an open walk of a merged delivery's repository and branch, at +// another commit, contains its merge: for a controller that does not name the merges a walk answers (issue 362), +// asked once per pair, the answer kept on the delivery. +func (h *Holder) containsAsks(d *Delivery, now time.Time) []func() { + var asks []func() + for _, s := range h.walks { + w := s.walk + if !h.mayContain(w, d) { + continue + } + if _, known := d.ContainedIn[w.Commit]; known { + continue + } + if a := h.askContains(d, w.Commit, now); a != nil { + asks = append(asks, a) + } + } + return asks +} + +// batchOf is the batch a merged delivery's merge is in, when the controller said one; false for none. +func (h *Holder) batchOf(d *Delivery) (Walk, bool) { + if d.MergedAs == "" { + return Walk{}, false + } + for _, s := range h.walks { + if !s.walk.IsBatch() || s.walk.Delivery == nil { + continue + } + for _, m := range s.walk.Delivery.Merges { + if strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) { + return s.walk, true + } + } + } + return Walk{}, false +} + +// batchSaid is what a batch holding a delivery's merge waits for, in words. +func batchSaid(b Walk, now time.Time) string { + said := "its batch " + b.ID + ": " + b.State + bt := b.Delivery.Batch + switch { + case b.State == walkQueued && bt != nil && bt.Behind != "": + said += " behind " + bt.Behind + case b.State == walkAssembling && bt != nil && bt.ClosesAt != nil: + left := bt.ClosesAt.Sub(now).Round(time.Second) + if left < 0 { + left = 0 + } + said += fmt.Sprintf(", %s left", left) + if bt.AtMost != nil { + said += " (at the latest " + bt.AtMost.Local().Format("15:04:05") + ")" + } + } + return said + "; plan not yet calculated" +} + +// unwalkedWhy is why a merged delivery no walk answers is held: the open walks of its repository and that none +// answers its merge, or that the controller holds none — what happened, not what might have. +func (h *Holder) unwalkedWhy(d *Delivery) string { + var open []string + for _, s := range h.walks { + w := s.walk + if w.Release == nil && !w.IsBatch() && w.Open() && strings.EqualFold(w.Repository, d.Repository) { + open = append(open, fmt.Sprintf("%s at %s", w.ID, shortOf(w.Commit, 8))) + } + } + sort.Strings(open) + head := fmt.Sprintf("merged as %s, and no walk answers its merge within %s: ", shortOf(d.MergedAs, 8), unwalked) + if len(open) == 0 { + return head + "the controller holds no open walk of " + d.Repository + " — its merge was not heard, or " + + "nothing the mesh holds follows " + orNone(d.Base) + ": a person decides" + } + return head + "the open walk(s) of " + d.Repository + " — " + strings.Join(open, ", ") + " — name no merge of " + + "it and the forge did not say they contain it: a person decides" +} + +func orNone(s string) string { + if s == "" { + return "its branch" + } + return s +} + +// orphanLines are the waiting walks no live delivery holds, past the grace a merge is given to be heard, as +// stalled lines in the state Unheld: nothing would ever give such a walk its word (issue 362). Under the lock. +func (h *Holder) orphanLines(now time.Time) []StalledLine { + var out []StalledLine + for id, first := range h.orphans { + s, known := h.walks[id] + if !known || now.Sub(first) < matchGrace { + continue + } + w := s.walk + out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, + For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), + H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", + Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ + "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ + "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) + } + return out +} + +// Unheld is a waiting walk no live delivery holds: not a state of the table — only what a stalled line says. +const Unheld State = "unheld" + +// seeOrphans keeps when each waiting walk no live delivery holds was first seen so, and forgets the rest; a +// walk still unmatched waits for its merge or its adoption instead. Under the lock. +func (h *Holder) seeOrphans(now time.Time) { + for id, s := range h.walks { + w := s.walk + _, unmatched := h.unmatched[id] + if unmatched || w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.heldWalk(w) { + delete(h.orphans, id) + continue + } + if _, was := h.orphans[id]; !was { + h.orphans[id] = now + h.Logf("[mesh-delivery] the walk %s waits for this owner's word and no live delivery holds it: said once "+ + "it is %s so", id, matchGrace) + } + } + for id := range h.orphans { + if _, known := h.walks[id]; !known { + delete(h.orphans, id) + } + } +} + +// forgetWalks drops a walk record no longer open after keptWalkFor, and a batch heard of that long past its +// window's latest close. Under the lock. +func (h *Holder) forgetWalks(now time.Time) { + for id, s := range h.walks { + w := s.walk + switch { + case w.IsBatch(): + end := s.at + if w.Delivery != nil && w.Delivery.Batch != nil && w.Delivery.Batch.AtMost != nil && w.Delivery.Batch.AtMost.After(end) { + end = *w.Delivery.Batch.AtMost + } + if now.Sub(end) > 3*time.Hour { + delete(h.walks, id) + } + case !w.Open() && now.Sub(s.at) > keptWalkFor: + delete(h.walks, id) + } + } +} + +// --- the word, once per walk ------------------------------------------------------------------------------ + +// turnWant is a walk whose turn came for some of the deliveries it carries, and why. +type turnWant struct { + ids []string + whys []string +} + +// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per +// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). +// Under the lock. +func (h *Holder) words() []func() { + walks := make([]string, 0, len(h.turning)) + for walk := range h.turning { + walks = append(walks, walk) + } + sort.Strings(walks) + var asks []func() + for _, walk := range walks { + want := h.turning[walk] + var waiting []string + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.ID != walk || containsString(want.ids, d.ID) { + continue + } + waiting = append(waiting, fmt.Sprintf("%s (%s)", d.ID, d.State)) + } + if len(waiting) > 0 { + sort.Strings(waiting) + h.Logf("[mesh-delivery] the walk %s is not let go yet: %s carried by it not at its turn", walk, + strings.Join(waiting, ", ")) + continue + } + ids, why := append([]string(nil), want.ids...), strings.Join(want.whys, "; ") + if h.asking[walk] { + continue // its word is being asked + } + h.asking[walk] = true + walk := walk + asks = append(asks, func() { + err := h.Controller.Deliver(walk, why) + h.mu.Lock() + defer h.mu.Unlock() + delete(h.asking, walk) + if err != nil { + h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) + return + } + for _, id := range ids { + d := h.deliveries[id] + if d == nil { + continue + } + d.LetGo, d.LetGoWhy = h.Now(), why + if d.Walk != nil && d.Walk.ID == walk { + d.Walk.Waits, d.Walk.GoBy = false, byOwner + } + h.settleAndKeep(d) + } + }) + } + h.turning = nil + return asks +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/effects.go b/modules/mesh-delivery/cmd/mesh-delivery/effects.go index 84780839..02d6694b 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/effects.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/effects.go @@ -108,7 +108,11 @@ func statusOf(d *Delivery) (string, string) { case Retired: return "success", "history: merged before this owner delivered anything, never walked" case Covered: - return "success", clip("covered: a later merge that contains it was delivered: "+d.CoveredBy, 140) + by := d.CoveredBy + if by == "" { + by = d.Carrier + } + return "success", clip("covered: a later merge that contains it was delivered: "+by, 140) } return "pending", string(d.State) } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder.go b/modules/mesh-delivery/cmd/mesh-delivery/holder.go index 5cbe008c..ce098104 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder.go @@ -45,6 +45,13 @@ type Holder struct { refusals []string // unchecked are the open pull requests' heads seen without a merge check (unchecked.go). unchecked map[string]unchecked + // walks are the walks' and batches' last records (answers.go); orphans the waiting walks no live delivery + // holds, by when that was first seen; turning the walks whose turn came this tick, for some deliveries. + walks map[string]seenWalk + orphans map[string]time.Time + turning map[string]*turnWant + // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. + asking map[string]bool } // unmatched is a walk no delivery is known for yet: its merge not heard, or merged before this owner held. @@ -86,6 +93,9 @@ func (h *Holder) init() { h.askTried = map[string]time.Time{} h.walkAsked = map[string]time.Time{} h.unchecked = map[string]unchecked{} + h.walks = map[string]seenWalk{} + h.orphans = map[string]time.Time{} + h.asking = map[string]bool{} h.started = h.Now() } @@ -401,13 +411,8 @@ func (h *Holder) PullMerged(p PullEvent) { } else { d.MergedAt = now } - // A walk already heard of for this commit. - for id, u := range h.unmatched { - if strings.EqualFold(u.walk.Repository, d.Repository) && sameCommit(u.walk.Commit, d.MergedAs) { - h.linkWalk(d, u.walk) - delete(h.unmatched, id) - } - } + // A walk already heard of that answers this merge: its own, or one that names it. + h.linkAnswering(d) // A merge long past heard only now, from the forge's word — replayed from the bus's history when this owner // first held the seat — is history: it reached the machines before this owner was there to deliver it, and // holding it for a person asks a person about the past (novox/hq issue 313). @@ -417,7 +422,7 @@ func (h *Holder) PullMerged(p PullEvent) { } ts = append(ts, settle(d, Facts{Now: now})...) if d.State == Held && d.HeldWhy == "" { - d.HeldWhy = "merged without a passing check: only a person decides that it goes on" + d.HeldWhy, d.Unwalked = "merged without a passing check: only a person decides that it goes on", false } h.moved(d, ts) h.closeGroupOf(d) @@ -562,8 +567,17 @@ func (h *Holder) walkMoved(w Walk) { h.creditBacklog(w) return } - d := h.deliveryWalking(w) - if d == nil { + if old, seen := h.walks[w.ID]; !seen || w.Revision >= old.walk.Revision { + h.walks[w.ID] = seenWalk{walk: w, at: h.Now()} + } + if w.IsBatch() { + return // a batch is shown, never linked, adopted or let go (ADR 0276) + } + links := h.deliveriesWalking(w) + if len(links) == 0 { + if h.accounted(w) { + return // it answers only deliveries that ended or follow a newer walk + } if _, seen := h.unmatched[w.ID]; !seen { h.unmatched[w.ID] = unmatched{walk: w, first: h.Now()} } else { @@ -575,43 +589,72 @@ func (h *Holder) walkMoved(w Walk) { } return } - if d.Walk != nil && d.Walk.ID == w.ID && w.Revision < d.Walk.Revision { - return // an older record arriving late + delete(h.unmatched, w.ID) + moved := map[string]bool{} + for _, a := range links { + d := a.d + if d.Walk != nil && d.Walk.ID == w.ID && w.Revision < d.Walk.Revision { + continue // an older record arriving late + } + h.linkWalk(d, w, a.carried) + h.settleAndKeep(d) + moved[d.ID] = true } - h.linkWalk(d, w) - h.settleAndKeep(d) // What its sends carried of other deliveries' builds. for _, m := range w.Modules { if m != nil && m.Gate != nil { - h.creditCarried(m.Gate, w.ID, d.ID) + h.creditCarried(m.Gate, w.ID, moved) } } } -// deliveryWalking is the delivery a walk walks: the one that landed as its commit, or that it walked before. -func (h *Holder) deliveryWalking(w Walk) *Delivery { - for _, d := range h.deliveries { - if d.Walk != nil && d.Walk.ID == w.ID { - return d - } +// linkWalk records a walk on its delivery: what it is at, the commit it carries for it, and the machine steps +// its record says — each moved only as the step table allows. A delivery that follows a new walk takes its +// steps as that walk's, a new journey; and one held because no walk answered it is lifted. +func (h *Holder) linkWalk(d *Delivery, w Walk, carried string) { + switching := d.Walk != nil && d.Walk.ID != w.ID + if switching { + h.Logf("[mesh-delivery] %s follows the walk %s now (it followed %s, %s)", d.ID, w.ID, d.Walk.ID, d.Walk.State) + d.LetGo, d.LetGoWhy = time.Time{}, "" // a new walk waits for a word of its own } - for _, d := range h.deliveries { - if strings.EqualFold(d.Repository, w.Repository) && sameCommit(d.MergedAs, w.Commit) { - return d - } - } - return nil -} - -// linkWalk records a walk on its delivery: what it is at, and the machine steps its record says — each -// moved only as the step table allows. -func (h *Holder) linkWalk(d *Delivery, w Walk) { d.Walk = w.Seen() + if carried != "" { + d.AnsweredBy = carried + } + d.Carrier = "" + if d.carriedLater() { + d.Carrier = fmt.Sprintf("merged as %s, carried by the walk %s", shortOf(d.AnsweredBy, 12), w.ID) + for _, o := range h.deliveries { + if o.ID != d.ID && strings.EqualFold(o.Repository, d.Repository) && sameCommit(o.MergedAs, d.AnsweredBy) { + d.Carrier = fmt.Sprintf("%s (merged as %s into %s, walk %s)", o.ID, shortOf(d.AnsweredBy, 8), orNone(d.Base), w.ID) + } + } + } + if d.State == Held && d.Unwalked && w.Open() { + d.HeldWhy = "" + d.Lifted = fmt.Sprintf("the walk %s, which answers its merge, opened", w.ID) + } + d.Unwalked = false // a walk answers it for _, s := range StepsOf(w) { + if switching { + h.restep(d, s) + continue + } h.step(d, s) } } +// restep puts a step of a walk a delivery newly follows, whatever the walk before said of that machine. +func (h *Holder) restep(d *Delivery, s Step) { + for i, old := range d.Steps { + if old.Machine == s.Machine && old.Module == s.Module { + d.Steps[i] = s + return + } + } + d.Steps = append(d.Steps, s) +} + // step moves one machine's step, or refuses a move the step table does not hold; it says whether it moved. func (h *Holder) step(d *Delivery, s Step) bool { for i, old := range d.Steps { @@ -647,7 +690,7 @@ func (h *Holder) creditBacklog(w Walk) { } } if w.Release.Gate != nil { - h.creditCarried(w.Release.Gate, w.ID, "") + h.creditCarried(w.Release.Gate, w.ID, nil) } if w.State == walkDone { h.creditReleased(w) @@ -704,7 +747,7 @@ func (h *Holder) askWalk(id string, now time.Time) func() { } // creditCarried gives the delivery that published each carried build its machine step. -func (h *Holder) creditCarried(g *WalkGate, walk, except string) { +func (h *Holder) creditCarried(g *WalkGate, walk string, except map[string]bool) { state := StepJudging switch { case g.Rollback == "rolled-back": @@ -716,7 +759,7 @@ func (h *Holder) creditCarried(g *WalkGate, walk, except string) { } for _, c := range g.Carried { for _, d := range h.deliveries { - if d.ID == except || d.MergedAs == "" || !sameCommit(d.MergedAs, c.To) { + if except[d.ID] || d.MergedAs == "" || !sameCommit(d.MergedAs, c.To) { continue } if h.step(d, Step{Machine: c.Node, Module: c.Module, Build: c.Build, State: state, Since: h.Now(), @@ -763,11 +806,19 @@ func (h *Holder) plan() []func() { now := h.Now() var asks []func() + h.forgetWalks(now) + // Walks no merge was heard for: made deliveries of their own once their merge had time to be heard. for id, u := range h.unmatched { - if d := h.deliveryWalking(u.walk); d != nil { - h.linkWalk(d, u.walk) - h.settleAndKeep(d) + if links := h.deliveriesWalking(u.walk); len(links) > 0 { + for _, a := range links { + h.linkWalk(a.d, u.walk, a.carried) + h.settleAndKeep(a.d) + } + delete(h.unmatched, id) + continue + } + if h.accounted(u.walk) { delete(h.unmatched, id) continue } @@ -813,6 +864,39 @@ func (h *Holder) plan() []func() { } } + // Merged with no walk yet: tied to the newest walk heard that answers it — by name, by its own commit, or by + // the forge's word that the walk's commit contains it, asked here (novox/hq issue 362). + for _, d := range h.deliveries { + if h.linkAnswering(d) { + h.settleAndKeep(d) + continue + } + asks = append(asks, h.containsAsks(d, now)...) + } + + // Following a walk that was taken over: the walk that took it over, read as soon as it is heard of, asked for + // by its id when it is not. + for _, d := range h.deliveries { + if d.State.Final() || d.Walk == nil || d.Walk.TakenOverBy == "" { + continue + } + if s, known := h.walks[d.Walk.TakenOverBy]; known { + if c, ok := walkAnswers(s.walk, d); ok { + h.linkWalk(d, s.walk, c) + } else { + // The walk that took its own over does not answer it: superseded, as before ADR 0276. + h.Logf("[mesh-delivery] %s: the walk %s, which took %s over, does not answer its merge", d.ID, + d.Walk.TakenOverBy, d.Walk.ID) + d.Walk.TakenOverBy = "" + } + h.settleAndKeep(d) + continue + } + if a := h.askWalk(d.Walk.TakenOverBy, now); a != nil { + asks = append(asks, a) + } + } + // Held, or merged with no walk opened for it: ended by what is already true of it, never asked of a person // (novox/hq issue 319). A plan that moves nothing is delivered; one whose every module a later merge into the // same trunk delivered is covered. Read again each tick, so a delivery read back after a restart, or held @@ -853,13 +937,13 @@ func (h *Holder) plan() []func() { // Merged, and no walk opened for it after the walks were read again: never silent — held, saying so. for _, d := range h.deliveries { if h.unwalkedPast(d, now) { - d.HeldWhy = "merged, and the controller opened no walk for it within " + unwalked.String() + " — nothing it " + - "holds follows that branch, or the merge was not heard: a person decides" + d.HeldWhy, d.Unwalked = h.unwalkedWhy(d), true h.settleAndKeep(d) } } // Groups: membership, order, composed check, and their turns. + h.turning = map[string]*turnWant{} grouped := map[string]bool{} for _, g := range h.groups { members := h.membersOf(g) @@ -874,10 +958,13 @@ func (h *Holder) plan() []func() { if grouped[d.ID] || d.Group != "" && h.groups[d.Group] != nil { continue } - if a := h.turn(d, "its turn: a delivery in no group"); a != nil { - asks = append(asks, a) - } + h.turn(d, "its turn: a delivery in no group") } + // The word, once per walk, when every live delivery it carries is at its turn. + asks = append(asks, h.words()...) + + // Waiting walks no live delivery holds: listed by `stalled` (issue 362). + h.seeOrphans(now) // Final deliveries past their keeping, once nothing is owed for them. for id, d := range h.deliveries { @@ -893,6 +980,9 @@ func (h *Holder) plan() []func() { // unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read // again after it: what holds it for a person, unless something already ends it. func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { + if _, batched := h.batchOf(d); batched { + return false // its batch is assembling or queued: a walk is not owed yet (ADR 0276) + } return d.State == Ready && d.MergedAs != "" && d.Walk == nil && !d.MovesNothing() && d.HeldWhy == "" && now.Sub(d.MergedAt) > unwalked && h.reconciled.After(d.MergedAt.Add(unwalked)) } @@ -1075,11 +1165,18 @@ func (h *Holder) adopt(w Walk) { d := &Delivery{ID: IDOf(w.Repository, w.Commit), Repository: strings.ToLower(w.Repository), Commit: w.Commit, Base: w.Branch, MergedAs: w.Commit, Created: now} if old := h.deliveries[d.ID]; old != nil { - h.linkWalk(old, w) + if old.State.Final() { + // Never revived: an ended delivery is not the walk's; the walk, held by nothing, is listed by + // `stalled` (issue 362). + h.Logf("[mesh-delivery] the walk %s is at %s, whose delivery has ended (%s): not adopted by it", w.ID, + shortOf(w.Commit, 12), old.State) + return + } + h.linkWalk(old, w, w.Commit) h.settleAndKeep(old) return } - h.linkWalk(d, w) + h.linkWalk(d, w, w.Commit) var t Transition var err error if d.Walk.Waits { @@ -1098,7 +1195,9 @@ func (h *Holder) adopt(w Walk) { h.settleAndKeep(d) } -// turn is the ask that lets a published delivery's waiting walk start; nil when it is not its turn. +// turn says a published delivery's waiting walk may start as far as that delivery goes; nil when it is not its +// turn. The word itself is asked once per walk by words, when every live delivery the walk carries is at its turn +// (ADR 0276 decision 5). What it answers is only whether this delivery's turn came. func (h *Holder) turn(d *Delivery, why string) func() { if d.Walk == nil || !d.Walk.Waits || !d.LetGo.IsZero() { return nil @@ -1106,25 +1205,22 @@ func (h *Holder) turn(d *Delivery, why string) func() { if d.State != Published && d.State != Delivering { return nil } - id, walk := d.ID, d.Walk.ID - return func() { - err := h.Controller.Deliver(walk, why) - h.mu.Lock() - defer h.mu.Unlock() - d := h.deliveries[id] - if d == nil { - return - } - if err != nil { - h.Logf("[mesh-delivery] %s's walk %s could not be let go yet: %v", id, walk, err) - return - } - d.LetGo, d.LetGoWhy = h.Now(), why - if d.Walk != nil && d.Walk.ID == walk { - d.Walk.Waits, d.Walk.GoBy = false, byOwner - } - h.settleAndKeep(d) + if d.Walk.State != walkBuilding && d.Walk.State != walkRolling { + return nil // a walk superseded, done or failed is no walk to start } + if h.turning == nil { + h.turning = map[string]*turnWant{} + } + want := h.turning[d.Walk.ID] + if want == nil { + want = &turnWant{} + h.turning[d.Walk.ID] = want + } + if !containsString(want.ids, d.ID) { + want.ids = append(want.ids, d.ID) + want.whys = append(want.whys, d.ID+": "+why) + } + return func() {} } // membersOf are a group's current members, by id order. @@ -1263,13 +1359,20 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { // Composed for these very heads: a verdict on another set of heads is no verdict on these. composed := g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") && sameMembers(g.Check.Commits, commitsOf(members)) + // A member's turn registered, the members after it that the same walk carries take theirs too: one walk is let + // go once, for all of them (ADR 0276 decision 5). A member on another walk, or not at its turn, waits. + var asks []func() + sharing := "" for i, m := range ordered { d := byID[m.ID] - switch { - case d.State.Reached(): + if d.State.Reached() { continue + } + if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { + return asks + } + switch { case d.State == Failed || d.State == Stopped || d.State == Superseded: - var asks []func() for _, later := range ordered[i+1:] { if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { asks = append(asks, a) @@ -1286,32 +1389,35 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() { h.settleAndKeep(d) if d.State == Published { if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil { - return []func(){a} + asks, sharing = append(asks, a), d.Walk.ID + continue } } } - return nil + return asks case d.State == Published: // Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316). if !composed && d.HeldWhy == "" && g.Check != nil && g.Check.Verdict == "" && sameMembers(g.Check.Commits, commitsOf(members)) { - return nil + return asks } if !composed && d.HeldWhy == "" { d.HeldWhy = fmt.Sprintf("its group %s's composed check did not pass for the heads that merged; a person "+ "decides that it goes on", g.ID) + d.Unwalked = false h.settleAndKeep(d) - return nil + return asks } if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil { - return []func(){a} + asks, sharing = append(asks, a), d.Walk.ID + continue } - return nil + return asks default: - return nil // not on the trunk yet, held, or delivering: the members after it wait + return asks // not on the trunk yet, held, or delivering: the members after it wait } } - return nil + return asks } // stopInGroup stops a member after one that failed, and its walk if it has one open. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go index dbc9ae79..e2ccccae 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/holder_test.go @@ -162,7 +162,8 @@ func TestAFailedGateIsRolledBackAndTheDeliveryFails(t *testing.T) { } } -// A newer merge's walk took over this one's: superseded, from published or delivering. +// A walk superseded that names no walk taking it over (a record from before ADR 0276): superseded, from +// published or delivering. One that names its successor is followed (TestADeliveryFollowsTheWalkThatTookItsOver). func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) { w := newWorld(t) id := IDOf("novox/app", head) @@ -172,7 +173,7 @@ func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) { walk := aWalk("plan-1", "novox/app", merge, true) w.h.WalkMoved(w.ctl.put(walk)) walk.State = walkSuperseded - walk.Note = "superseded at tier 0 by plan-2" + walk.Note = "superseded" w.ctl.down = true // its word could not be given meanwhile w.settleAll() w.h.WalkMoved(w.ctl.put(walk)) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go b/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go new file mode 100644 index 00000000..c37f72f3 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/issue362_test.go @@ -0,0 +1,434 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// novox/hq issue 362, ADR 0276: two merges of one repository 18 seconds apart, walked by one walk at the later +// merge's commit, which superseded the first. Every delivery the walk answers follows it and ends with it; no +// walk is tied to a delivery that ended; and what cannot move is said at once. + +// A controller that names the merges a walk answers (ADR 0276): the forge is not asked, and the earlier merge is +// tied at once. +func TestAWalkThatNamesItsMergesIsFollowedByEachOfThem(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + for _, id := range []string{IDOf(cat, headCC), IDOf(cat, headDu)} { + if d := w.delivery(id); d.State != Published || d.Walk == nil || d.Walk.ID != "plan-9" { + t.Fatalf("%s is %s on %v", id, d.State, d.Walk) + } + } + w.settleAll() + if len(w.forge.containsAsks) != 0 { + t.Fatalf("the forge was asked %v for a walk that names its merges", w.forge.containsAsks) + } + if len(w.ctl.delivers) != 1 { + t.Fatalf("the word was asked %v", w.ctl.delivers) + } +} + +// A walk that failed marks no merge delivered: the merge it carried its own commit for fails, and the earlier +// merge it carried a later commit for waits for a walk of its own. +func TestAFailedWalkMarksNoMergeItCarriedDelivered(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk("plan-9") + walk.State, walk.Note = walkFailed, "dunst failed its gate on anchor" + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(IDOf(cat, headDu)); s != Failed { + t.Fatalf("dunst is %s", s) + } + cc := w.delivery(IDOf(cat, headCC)) + if cc.State != Published || cc.State.Reached() { + t.Fatalf("claude-code is %s", cc.State) + } + // Its own walk, when the controller opens one, is followed. + own := catWalk("plan-10", mergeCC, atDu.Add(time.Hour), "claude-code") + w.h.WalkMoved(w.ctl.put(own)) + w.settleAll() + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-10"), w.now))) + w.settleAll() + if s := w.state(IDOf(cat, headCC)); s != Delivered { + t.Fatalf("claude-code on its own walk is %s", s) + } +} + +// Never revived: a walk at the commit of a delivery that ended is not tied to it, not adopted, and not let go — +// it is listed by `stalled` as a walk no live delivery holds. +func TestAWalkIsNeverTiedToADeliveryThatEnded(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(supersededBy(first, ""))) + // A record from before ADR 0276: superseded, naming no walk — the delivery ends superseded. + sup := w.ctl.walk("plan-1") + sup.Note = "superseded" + w.h.WalkMoved(w.ctl.put(sup)) + idDu := IDOf(cat, headDu) + if s := w.state(idDu); s != Superseded { + t.Fatalf("dunst is %s", s) + } + orphan := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "dunst") + w.h.WalkMoved(w.ctl.put(orphan)) + w.settleAll() + w.later(3 * time.Minute) + w.settleAll() + if d := w.delivery(idDu); d.State != Superseded || d.Walk.ID != "plan-1" { + t.Fatalf("the ended delivery is %s on %s", d.State, d.Walk.ID) + } + for _, asked := range w.ctl.delivers { + if asked == "plan-2" { + t.Fatal("a walk no live delivery holds was let go") + } + } + var found bool + for _, l := range w.h.Stalled() { + if l.State == Unheld && l.Walk == "plan-2" { + found = true + } + } + if !found { + t.Fatalf("stalled does not list the walk no delivery holds: %+v", w.h.Stalled()) + } +} + +// A merge no walk answers is held after ten minutes with the true reason, and listed by `stalled` at once; a walk +// that answers it later lifts it. +func TestAMergeNoWalkAnswersIsHeldTrulyAndSaidAtOnce(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + // An open walk of the repository that moves something else: it does not answer this merge. + w.h.WalkMoved(w.ctl.put(catWalk("plan-5", "eeeeeeeeeeee5555", atCC.Add(time.Minute), "dunst"))) + w.later(11 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + id := IDOf(cat, headCC) + d := w.delivery(id) + if d.State != Held || !strings.Contains(d.HeldWhy, "plan-5") || !strings.Contains(d.HeldWhy, "name no merge") { + t.Fatalf("it is %s: %q", d.State, d.HeldWhy) + } + lines := w.h.Stalled() + if len(lines) != 1 || lines[0].ID != id || lines[0].Bound != "0s" { + t.Fatalf("stalled lists %+v", lines) + } + // The controller opens a walk that names it. + walk := catWalk("plan-6", mergeCC, w.now, "claude-code") + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(id); s != Delivering || len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-6" { + t.Fatalf("with its walk it is %s, let go %v", s, w.ctl.delivers) + } +} + +// One word per walk, and none while a delivery it carries is not at its turn. +func TestAWalkIsLetGoOnlyWhenEveryDeliveryItCarriesIsAtItsTurn(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(cc) + w.h.PullUpdated(du) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + // dunst merges unchecked: held for a person. + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.PullMerged(mergedAt(du, mergeDu, atDu)) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if s := w.state(IDOf(cat, headDu)); s != Held { + t.Fatalf("dunst is %s", s) + } + if len(w.ctl.delivers) != 0 { + t.Fatalf("the walk was let go %v while it carries a held delivery", w.ctl.delivers) + } + if _, err := w.h.Release(IDOf(cat, headDu), "checked by hand", "jochen"); err != nil { + t.Fatal(err) + } + w.settleAll() + if len(w.ctl.delivers) != 1 { + t.Fatalf("released, the word was asked %v", w.ctl.delivers) + } +} + +// A merge in a batch still assembling is said so, is not held for a person, and nothing is let go. +func TestAMergeInAnAssemblingBatchIsShownAndNotHeld(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + closes, atMost := atCC.Add(90*time.Second), atCC.Add(10*time.Minute) + batch := Walk{ID: "plan-b1", Repository: cat, Branch: "main", Commit: mergeCC, Created: atCC, State: walkAssembling, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: cat, Commit: mergeCC}}, + Batch: &WalkBatch{ClosesAt: &closes, AtMost: &atMost}}} + w.h.WalkMoved(w.ctl.put(batch)) + w.later(48 * time.Second) + w.settleAll() + id := IDOf(cat, headCC) + var line Line + for _, l := range w.h.Deliveries("", "", "", true) { + if l.ID == id { + line = l + } + } + if !strings.Contains(line.Waits, "plan-b1: assembling, 42s left") || !strings.Contains(line.Waits, "plan not yet calculated") { + t.Fatalf("it waits for %q", line.Waits) + } + // Queued behind a walk for longer than a walk opens within: still not held. + batch = w.ctl.walk("plan-b1") + batch.State, batch.Delivery.Batch.Behind = walkQueued, "plan-7" + w.h.WalkMoved(w.ctl.put(batch)) + w.later(20 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + if d := w.delivery(id); d.State != Ready || d.Walk != nil { + t.Fatalf("a merge in a queued batch is %s on %v", d.State, d.Walk) + } + if len(w.ctl.delivers) != 0 { + t.Fatalf("a batch was let go: %v", w.ctl.delivers) + } + // Cut: the batch is a walk now, under its id. + batch.State, batch.Delivery.Batch = walkBuilding, nil + batch.Tiers, batch.Modules = [][]string{{"claude-code"}}, map[string]*WalkModule{"claude-code": {}} + w.h.WalkMoved(w.ctl.put(batch)) + w.settleAll() + if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-b1" { + t.Fatalf("cut, it is %s on %v", d.State, d.Walk) + } +} + +// An older record of the walk a delivery no longer follows, arriving late, does not take it back. +func TestALateRecordOfAnOlderWalkDoesNotTakeADeliveryBack(t *testing.T) { + w := twoMerges(t) + w.forge.contains[mergeCC+" in "+mergeDu] = true + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))) + w.h.WalkMoved(first) // the older record, again + if d := w.delivery(IDOf(cat, headDu)); d.Walk.ID != "plan-2" { + t.Fatalf("dunst went back to %s", d.Walk.ID) + } +} + +// A walk no merge was heard for, at the commit of a delivery that ended, is not adopted by that delivery: it is +// listed by `stalled` as a walk no live delivery holds. +func TestAnUnheardWalkIsNotAdoptedByADeliveryThatEnded(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.PullClosed(cc) + id := IDOf(cat, headCC) + if s := w.state(id); s != Stopped { + t.Fatalf("closed unmerged it is %s", s) + } + w.h.WalkMoved(w.ctl.put(catWalk("plan-3", headCC, atCC.Add(time.Minute), "claude-code"))) + w.later(3 * time.Minute) + w.settleAll() + w.later(3 * time.Minute) + w.settleAll() + if d := w.delivery(id); d.State != Stopped || d.Walk != nil { + t.Fatalf("the stopped delivery is %s on %v", d.State, d.Walk) + } + var found bool + for _, l := range w.h.Stalled() { + found = found || l.State == Unheld && l.Walk == "plan-3" + } + if !found || len(w.ctl.delivers) != 0 { + t.Fatalf("stalled lists %+v; let go %v", w.h.Stalled(), w.ctl.delivers) + } +} + +// Two members of one delivery group carried by one walk: let go once, for both (review of #171: the group's turn +// stopped at its first member, and the walk waited for the second for ever). +func TestAWalkCarryingTwoMembersOfAGroupIsLetGoOnce(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + a := pr("novox/a", 1, "aaaaaaaaaaaa0001", "feat/y", "x.go") + b := pr("novox/b", 2, "bbbbbbbbbbbb0002", "feat/y", "y.go") + w.h.PullUpdated(a) + w.h.PullUpdated(b) + w.h.Checked(verdictMoving("novox/a", 1, "aaaaaaaaaaaa0001", "app-a")) + w.h.Checked(verdictMoving("novox/b", 2, "bbbbbbbbbbbb0002", "app-b")) + w.settleAll() + g := w.h.groups["feat/y"] + if g == nil || g.Check == nil { + t.Fatalf("the group is %+v", g) + } + w.h.Checked(CheckedEvent{Group: "feat/y", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}) + w.h.PullMerged(mergedAt(a, "aaaaaaaaaaaa1111", atCC)) + w.h.PullMerged(mergedAt(b, "bbbbbbbbbbbb2222", atDu)) + batch := Walk{ID: "plan-batch", Repository: "novox/a", Branch: "main", Commit: "aaaaaaaaaaaa1111", + Created: atDu.Add(time.Minute), State: walkBuilding, Tiers: [][]string{{"app-a"}, {"app-b"}}, + Modules: map[string]*WalkModule{"app-a": {}, "app-b": {}}, + Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{ + {Repository: "novox/a", Commit: "aaaaaaaaaaaa1111"}, {Repository: "novox/b", Commit: "bbbbbbbbbbbb2222"}}}} + w.h.WalkMoved(w.ctl.put(batch)) + w.settleAll() + if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-batch" { + t.Fatalf("the word was asked %v, want once for plan-batch", got) + } + for _, id := range []string{IDOf("novox/a", "aaaaaaaaaaaa0001"), IDOf("novox/b", "bbbbbbbbbbbb0002")} { + if s := w.state(id); s != Delivering { + t.Fatalf("%s is %s", id, s) + } + } +} + +// A walk that names its merges answers only those: a delivery whose walk it took over and which it does not name +// is not tied to it, and ends superseded once that is known. +func TestAWalkThatNamesItsMergesAnswersOnlyThose(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + other := catWalk("plan-2", "ffffffffffff9999", atDu.Add(6*time.Second), "dunst") + other.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: "ffffffffffff9999"}} + w.h.WalkMoved(w.ctl.put(other)) + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + w.settleAll() + d := w.delivery(IDOf(cat, headDu)) + if d.State != Superseded || d.Walk.ID != "plan-1" { + t.Fatalf("dunst is %s on %s", d.State, d.Walk.ID) + } + for _, asked := range w.ctl.delivers { + if asked == "plan-2" { + t.Fatal("a walk that does not name dunst was let go for it") + } + } +} + +// The forge's containment ties a merge to a walk only when the walk moves everything the merge moves and was made +// for a later merge. +func TestContainmentAloneDoesNotTieAMerge(t *testing.T) { + for _, c := range []struct { + name string + modules []string + created time.Time + names bool + }{ + {"the later walk moves only its own module", []string{"dunst"}, atDu.Add(5 * time.Second), false}, + {"the walk was made for an earlier merge", []string{"claude-code", "dunst"}, atCC.Add(-time.Minute), false}, + {"the walk names its merges, and not this one", []string{"claude-code", "dunst"}, atDu.Add(5 * time.Second), true}, + } { + t.Run(c.name, func(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + w.h.PullUpdated(cc) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.forge.contains[mergeCC+" in "+mergeDu] = true + walk := catWalk("plan-2", mergeDu, c.created, c.modules...) + if c.names { + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeDu}} + } + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + if d := w.delivery(IDOf(cat, headCC)); d.Walk != nil || len(w.forge.containsAsks) != 0 { + t.Fatalf("it follows %v; the forge was asked %v", d.Walk, w.forge.containsAsks) + } + }) + } +} + +// After the walk that carried a later commit for it failed, a merge stays published — once — and is said within +// the ten minutes a walk opens within, naming what happened. +func TestAMergeWhoseCarryingWalkFailedWaitsOnceAndIsSaid(t *testing.T) { + w := twoMerges(t) + walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst") + walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}} + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + walk = w.ctl.walk("plan-9") + walk.State = walkFailed + w.h.WalkMoved(w.ctl.put(walk)) + w.settleAll() + id := IDOf(cat, headCC) + d := w.delivery(id) + rewalks := 0 + for _, tr := range d.Transitions { + if tr.Event == EvRewalk { + rewalks++ + } + } + if d.State != Published || rewalks != 1 { + t.Fatalf("it is %s after %d rewalk(s)", d.State, rewalks) + } + w.later(11 * time.Minute) + var line *StalledLine + for _, l := range w.h.Stalled() { + if l.ID == id { + l := l + line = &l + } + } + if line == nil || !strings.Contains(line.Says, "no walk of its own commit") { + t.Fatalf("stalled says %+v", line) + } +} + +// A delivery that ended is never tied to a walk after: not one that answers its merge, not the walk that took its +// own over. +func TestADeliveryThatEndedFollowsNoWalk(t *testing.T) { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC) + w.ctl.planned = planMoving(cat) + p := pr(cat, 103, "eeeeeeeeeeee0003", "docs/x", "README.md") + w.h.PullUpdated(p) + w.h.PullMerged(mergedAt(p, "eeeeeeeeeeee3333", atCC)) + w.settleAll() + nothing := IDOf(cat, "eeeeeeeeeeee0003") + if s := w.state(nothing); s != Delivered { + t.Fatalf("a merge moving nothing is %s", s) + } + w.h.WalkMoved(w.ctl.put(catWalk("plan-4", "eeeeeeeeeeee3333", atCC.Add(time.Minute), "app"))) + w.settleAll() + if d := w.delivery(nothing); d.Walk != nil { + t.Fatalf("a delivered merge was tied to %s", d.Walk.ID) + } + + w2 := twoMerges(t) + first := w2.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w2.h.WalkMoved(first) + idDu := IDOf(cat, headDu) + if _, err := w2.h.Stop(idDu, "not now", "jochen"); err != nil { + t.Fatal(err) + } + w2.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")) + w2.h.WalkMoved(w2.ctl.put(supersededBy(w2.ctl.walk("plan-1"), "plan-2"))) + w2.settleAll() + if d := w2.delivery(idDu); d.State != Stopped || d.Walk.ID != "plan-1" { + t.Fatalf("the stopped delivery is %s on %s", d.State, d.Walk.ID) + } +} + +// Today's controller, merges A then B minutes apart: B's walk took A's over before it started. A follows it and +// ends covered by B, rather than superseded (ADR 0276 decision 4). +func TestAnEarlierMergeFollowsTheWalkThatTookItsOver(t *testing.T) { + w := twoMerges(t) + first := w.ctl.put(catWalk("plan-1", mergeCC, atCC.Add(5*time.Second), "claude-code")) + w.h.WalkMoved(first) + w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))) + w.h.WalkMoved(w.ctl.put(supersededBy(w.ctl.walk("plan-1"), "plan-2"))) + w.settleAll() + idCC := IDOf(cat, headCC) + if d := w.delivery(idCC); d.Walk.ID != "plan-2" || len(w.ctl.delivers) != 1 { + t.Fatalf("claude-code follows %s; let go %v", d.Walk.ID, w.ctl.delivers) + } + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now))) + w.settleAll() + if s := w.state(idCC); s != Covered { + t.Fatalf("claude-code is %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 17004128..7104748c 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -68,7 +68,14 @@ type Delivery struct { // ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's // merge (gitea_contains): asked once per pair, and kept (ADR 0270). ContainedIn map[string]bool `json:"contained_in,omitempty"` - // Walk is the controller's walk of its trunk commit, as last read; Steps the machines it reached. + // AnsweredBy is the commit of its repository the walk it follows carries for it: its own merge, or a later merge + // of its branch that contains it (novox/hq ADR 0276). Unwalked says it is held because no walk answered it. + AnsweredBy string `json:"answered_by,omitempty"` + // Carrier says, for a merge a later one carries, which: its delivery and walk, in words. + Carrier string `json:"carrier,omitempty"` + Unwalked bool `json:"unwalked,omitempty"` + // Walk is the walk that delivers it, as last read: its own trunk commit's, or one that carries a later merge + // of its branch; Steps the machines it reached. Walk *WalkSeen `json:"walk,omitempty"` Steps []Step `json:"steps,omitempty"` // LetGo is when this owner asked the controller to start its walk, and why. @@ -166,6 +173,16 @@ type MachinePlan struct { Waits []string `json:"waits,omitempty"` } +// waitsForItsOwnWalk is whether it is published again after the walk that carried a later commit for it failed. +func (d *Delivery) waitsForItsOwnWalk() bool { + return d.State == Published && d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater() +} + +// carriedLater is whether the walk it follows carries a later merge of its branch for it, not its own. +func (d *Delivery) carriedLater() bool { + return d.AnsweredBy != "" && d.MergedAs != "" && !sameCommit(d.AnsweredBy, d.MergedAs) +} + // MovesNothing is whether the delivery's plan moves no module the mesh holds: nothing for a walk to do. A // module it adds is not moved: the controller builds and registers it and sends it nowhere, and opens no walk // for it (novox/hq issues 300 and 319). A delivery with no plan known is not said to move nothing. @@ -337,7 +354,7 @@ type WalkRelease struct { By string `json:"by,omitempty"` } -// WalkDelivery is what a walk waits for. +// WalkDelivery is what a walk waits for, and which merges it answers (novox/hq ADR 0276). type WalkDelivery struct { Awaits string `json:"awaits"` Go *time.Time `json:"go,omitempty"` @@ -345,6 +362,81 @@ type WalkDelivery struct { Why string `json:"why,omitempty"` Stopped string `json:"stopped,omitempty"` StoppedWhy string `json:"stopped_why,omitempty"` + // Merges are every merge the walk answers: each merge commit, its repository, and the commit of that + // repository the walk carries — the merge itself, or a later merge of its branch that contains it. Absent + // from a controller that does not say it: the walk then answers the merge of its own commit, and a merge + // the forge says its commit contains. + Merges []WalkMerge `json:"merges,omitempty"` + // TakenOverBy is, on a walk folded before it started, the walk that took it over and answers every merge + // it answered. + TakenOverBy string `json:"taken_over_by,omitempty"` + // Batch is set while the record is a batch, not yet a walk: its state `assembling` or `queued`. A batch, once + // cut, is the walk under the same id: its next record is `building`. A merge this owner reads in a batch is + // not held as unwalked while the batch is heard of. + Batch *WalkBatch `json:"batch,omitempty"` +} + +// WalkMerge is one merge a walk answers. +type WalkMerge struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + // Carried is the commit of the repository the walk carries for it: Commit itself, or a later merge of its + // branch that contains it. Empty means Commit. + Carried string `json:"carried,omitempty"` +} + +// WalkBatch is a batch's window: when it closes unless another merge arrives, when it closes at the latest, +// and, once closed, the walk it waits behind. +type WalkBatch struct { + ClosesAt *time.Time `json:"closes_at,omitempty"` + AtMost *time.Time `json:"at_most,omitempty"` + Behind string `json:"behind,omitempty"` +} + +// The states a batch's record is in before it is a walk (novox/hq ADR 0276). +const ( + walkAssembling = "assembling" + walkQueued = "queued" +) + +// IsBatch is whether the record is a batch still to be cut into a walk. +func (w Walk) IsBatch() bool { return w.State == walkAssembling || w.State == walkQueued } + +// supersededNote reads the walk that took a superseded walk over from its note, for a controller that does not +// say TakenOverBy: "superseded at tier N by plan-… (…)". +var supersededNote = regexp.MustCompile(`^superseded at tier \d+ by (plan-[0-9A-Za-z_-]+)`) + +// takenOverBy is the walk that took this one over, said or read from its note; empty for none. +func (w Walk) takenOverBy() string { + if w.State != walkSuperseded { + return "" + } + if w.Delivery != nil && w.Delivery.TakenOverBy != "" { + return w.Delivery.TakenOverBy + } + if m := supersededNote.FindStringSubmatch(w.Note); m != nil { + return m[1] + } + return "" +} + +// carriedFor is the commit the walk carries for a merge of a repository, and whether it answers that merge by +// what it says: the merge named among its merges, or its own commit. +func (w Walk) carriedFor(repository, merge string) (string, bool) { + if w.Delivery != nil { + for _, m := range w.Delivery.Merges { + if strings.EqualFold(m.Repository, repository) && sameCommit(m.Commit, merge) { + if m.Carried != "" { + return m.Carried, true + } + return m.Commit, true + } + } + } + if strings.EqualFold(w.Repository, repository) && sameCommit(w.Commit, merge) { + return w.Commit, true + } + return "", false } // WalkSeen is what a delivery keeps of its walk. @@ -363,6 +455,10 @@ type WalkSeen struct { Updated time.Time `json:"updated"` // Branch is the branch the walk's commit was merged into; empty in a record kept before it was read. Branch string `json:"branch,omitempty"` + // Created is when the controller made the walk: a delivery follows the newest walk that answers it. + Created time.Time `json:"created"` + // TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it. + TakenOverBy string `json:"taken_over_by,omitempty"` } // Seen is a walk as a delivery keeps it. @@ -376,6 +472,8 @@ func (w Walk) Seen() *WalkSeen { s.StoppedBy = d.Stopped } s.Branch = w.Branch + s.Created = w.Created + s.TakenOverBy = w.takenOverBy() return s } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go b/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go new file mode 100644 index 00000000..7e83f79e --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/replay362_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// Replay R362 (novox/hq issue 362, mesh-lab's register): two merges of the catalogue 18 seconds apart, walked by +// one walk at the later merge's commit, which superseded the first — as the controller of 2026-10-10 said it, with +// no `merges` and the successor named only in the superseded walk's note. Written with only what mesh-delivery +// had before its fix (and fakes_test.go, covered_test.go's helpers), so it can be laid over the commit before. + +const ( + cat = "novox/mesh-catalog" + headCC = "cccccccccccc1111" // claude-code's head + headDu = "dddddddddddd2222" // dunst's head + mergeCC = "553b71910000aaaa" // merged first, 00:20:52 + mergeDu = "48bda4750000bbbb" // merged 18 s later, 00:21:09: contains mergeCC +) + +var ( + atCC = time.Date(2026, 10, 9, 22, 20, 52, 0, time.UTC) + atDu = atCC.Add(17 * time.Second) +) + +// twoMerges is the night of issue 362 up to the walks: both pull requests checked, both merged. +func twoMerges(t *testing.T) *world { + w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second)) + cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") + du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") + w.h.PullUpdated(cc) + w.h.PullUpdated(du) + w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code")) + w.h.Checked(verdictMoving(cat, 102, headDu, "dunst")) + w.h.PullMerged(mergedAt(cc, mergeCC, atCC)) + w.h.PullMerged(mergedAt(du, mergeDu, atDu)) + return w +} + +// catWalk is a walk of the catalogue at a commit, waiting for its word, moving the modules given. +func catWalk(id, commit string, created time.Time, modules ...string) Walk { + w := Walk{ID: id, Repository: cat, Branch: "main", Commit: commit, Created: created, State: walkBuilding, + Tiers: [][]string{modules}, Modules: map[string]*WalkModule{}, Delivery: &WalkDelivery{Awaits: "mesh-delivery"}} + for _, m := range modules { + w.Modules[m] = &WalkModule{} + } + return w +} + +// done is a walk's record once every module passed on its first machine and was sent to the rest. +func done(w Walk, at time.Time) Walk { + w.State = walkDone + for name := range w.Modules { + judged := at + w.Modules[name] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &judged, SentAt: &judged, + Gate: &WalkGate{Machines: []string{"anchor"}, Verdict: "passed", JudgedAt: &judged}} + } + return w +} + +func supersededBy(old Walk, by string) Walk { + old.State = walkSuperseded + old.Note = "superseded at tier 0 by " + by + " (novox/mesh-catalog at 48bda475); dunst planned there again" + return old +} + +// The replay of issue 362, as today's controller says it: no `merges`, the successor named only in its note. In +// both orders the controller's two records can arrive in, the walk at the dunst commit delivers both merges +// without a person: claude-code covered, dunst delivered, the word asked once. +func TestReplay362(t *testing.T) { + for _, order := range []string{"newer walk first", "supersession first"} { + t.Run(order, func(t *testing.T) { + w := twoMerges(t) + w.forge.contains[mergeCC+" in "+mergeDu] = true + idCC, idDu := IDOf(cat, headCC), IDOf(cat, headDu) + first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst")) + w.h.WalkMoved(first) + if s := w.state(idDu); s != Published { + t.Fatalf("dunst with its walk opened is %s", s) + } + second := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst") + if order == "newer walk first" { + w.h.WalkMoved(w.ctl.put(second)) + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + } else { + w.ctl.put(second) // kept by the controller, said after + w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2"))) + } + w.settleAll() + for _, id := range []string{idCC, idDu} { + if d := w.delivery(id); d.State.Final() || d.State == Held { + t.Fatalf("%s is %s (%s)", id, d.State, d.HeldWhy) + } + } + if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-2" { + t.Fatalf("the word was asked %v, want once for plan-2", got) + } + for _, id := range []string{idCC, idDu} { + if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-2" { + t.Fatalf("%s is %s on %v", id, d.State, d.Walk) + } + } + // Ten minutes and a reconcile later nothing is held for a person. + w.later(11 * time.Minute) + if err := w.h.Reconcile(); err != nil { + t.Fatal(err) + } + w.settleAll() + w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now))) + w.settleAll() + cc, du := w.delivery(idCC), w.delivery(idDu) + if du.State != Delivered { + t.Fatalf("dunst is %s", du.State) + } + if cc.State != Covered || !strings.Contains(cc.Transitions[len(cc.Transitions)-1].Why, idDu) { + t.Fatalf("claude-code is %s: %+v", cc.State, cc.Transitions[len(cc.Transitions)-1]) + } + if len(w.h.Stalled()) != 0 { + t.Fatalf("stalled lists %+v", w.h.Stalled()) + } + if len(w.ctl.delivers) != 1 { + t.Fatalf("the word was asked %v", w.ctl.delivers) + } + }) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index e554cabf..ce8fa550 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -83,7 +83,8 @@ const ( EvHistory Event = "history" // made from the forge's word for a merge long past: history EvRetire Event = "retire" // a person's word that a held delivery is history, never to be walked EvCovered Event = "covered" // a later merge into the same trunk delivered every module it moves - EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed + EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed, or a walk now answers its merge + EvRewalk Event = "rewalk" // the walk that carried a later commit for it failed: it waits for a walk of its own commit ) // Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act. @@ -222,8 +223,11 @@ var Table = []Row{ }}, {From: []State{Published}, Event: EvSuperseded, To: Superseded, - Guard: "a newer delivery to the same trunk took over its walk", + Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)", Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" { + return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy) + } return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded") }}, {From: []State{Published}, Event: EvDone, To: Delivered, @@ -236,8 +240,14 @@ var Table = []Row{ return unless(d.HeldWhy != "" && d.MergedAs != "", "nothing holds it, or it is not on the trunk") }}, {From: []State{Published}, Event: EvGo, To: Delivering, - Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path", - Holds: func(d *Delivery, f Facts) error { return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") }}, + Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path " + + "— not a failed walk that carried a later commit for it, after which it waits for a walk of its own commit", + Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater() { + return errors.New("its walk, which carried a later commit for it, failed: it waits for a walk of its own commit") + } + return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") + }}, {From: []State{Held}, Event: EvGo, To: Delivering, Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " + "nobody, or a person's `plans go`", @@ -246,8 +256,8 @@ var Table = []Row{ "its walk was not started by a person or the controller's own path") }}, {From: []State{Held}, Event: EvLifted, To: Published, - Guard: "held only for its group's composed check, which has since passed or warned for the heads that merged: " + - "it is published again and waits for its turn", + Guard: "held only for its group's composed check, which has since passed or warned for the heads that merged, or " + + "held because no walk answered its merge, and one now does: it is published again and waits for its turn", Holds: func(d *Delivery, f Facts) error { return unless(d.Lifted != "" && d.HeldWhy == "" && d.MergedAs != "" && walkCanGoOn(d), "nothing lifted its hold") }, @@ -259,14 +269,26 @@ var Table = []Row{ }}, {From: []State{Delivering}, Event: EvSuperseded, To: Superseded, - Guard: "a newer delivery to the same trunk took over its walk", + Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)", Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" { + return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy) + } return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded") }}, {From: []State{Delivering}, Event: EvDone, To: Delivered, - Guard: "its walk is done: every machine of its deploy plan passed, or was left as its policy says", + Guard: "its walk is done, carrying its own commit: every machine of its deploy plan passed, or was left as its policy says", Holds: func(d *Delivery, f Facts) error { - return unless(d.Walk != nil && d.Walk.State == walkDone, "its walk is not done") + return unless(d.Walk != nil && d.Walk.State == walkDone && !d.carriedLater(), "its walk is not done, or carried a later commit for it") + }}, + {From: []State{Delivering}, Event: EvCovered, To: Covered, + Guard: "its walk is done, carrying a later merge of its branch that contains its merge: answered by its successor in " + + "one walk (novox/hq ADR 0276)", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.State == walkDone && d.carriedLater(), "its walk is not done, or carried its own commit") + }, + Says: func(d *Delivery) string { + return "covered: a later merge that contains it was delivered: " + d.Carrier }}, {From: []State{Delivering}, Event: EvStop, To: Stopped, Guard: "its walk was stopped through this owner", @@ -274,9 +296,21 @@ var Table = []Row{ return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy != "", "its walk was not stopped") }}, {From: []State{Delivering}, Event: EvFailed, To: Failed, - Guard: "its walk failed: a gate on a first machine (what it carried put back), a build, a machine", + Guard: "its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine", Holds: func(d *Delivery, f Facts) error { - return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "", "its walk did not fail") + return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && !d.carriedLater(), + "its walk did not fail, or carried a later commit for it") + }}, + {From: []State{Delivering}, Event: EvRewalk, To: Published, + Guard: "its walk failed carrying a later merge of its branch for it: a failed walk marks no merge delivered, and this " + + "merge waits, published, for a walk of its own commit (novox/hq ADR 0276 decision 3)", + Holds: func(d *Delivery, f Facts) error { + return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && d.carriedLater(), + "its walk did not fail, or carried its own commit") + }, + Says: func(d *Delivery) string { + return fmt.Sprintf("its walk %s, which carried %s for it, failed: it waits for a walk of its own commit", + d.Walk.ID, shortOf(d.AnsweredBy, 12)) }}, {From: notFinal, Event: EvStop, To: Stopped, Act: true, @@ -450,10 +484,31 @@ func boundSince(d *Delivery) time.Time { return d.Since } +// heldUnwalked is the bound of a delivery held because no walk answers its merge (novox/hq issue 362): none. It +// waited the ten minutes a walk opens within already, and nothing but a person moves it now. +var heldUnwalked = Bound{For: 0, Says: "merged, and no walk answers its merge: the operator's"} + +// rewalkBound is the bound of a merge whose walk carried a later commit for it and failed: a walk of its own +// commit opens within the ten minutes a walk opens within (ADR 0276 decision 3), or nothing but a person moves it. +var rewalkBound = Bound{For: unwalked, Says: "the walk that carried a later commit for it failed, and no walk of " + + "its own commit opened: the operator's"} + +// boundOf is a delivery's bound: its state's, or none for a delivery held because no walk answers it. +func boundOf(d *Delivery) (Bound, bool) { + if d.State == Held && d.Unwalked && d.Walk == nil { + return heldUnwalked, true + } + if d.waitsForItsOwnWalk() { + return rewalkBound, true + } + b, bounded := Bounds[d.State] + return b, bounded +} + // pastBound is whether a delivery has been where it is longer than its state's bound. func pastBound(d *Delivery, now time.Time) bool { - b, bounded := Bounds[d.State] - return bounded && !d.State.Final() && now.Sub(boundSince(d)) > b.For + b, bounded := boundOf(d) + return bounded && !d.State.Final() && (b.For == 0 || now.Sub(boundSince(d)) > b.For) } // StepState is where one machine stands in a delivering walk. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index 73cbde8e..072def6c 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -65,6 +65,14 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { } case EvCovered: d.MergedAs, d.CoveredBy = "aaaa1111", "delivered with novox/app@bbbb (merged as bbbb2222, walk plan-2): app" + if r.From[0] == Delivering { + // Answered by its successor in one walk (ADR 0276): the walk done, carrying a later merge. + d.CoveredBy, d.AnsweredBy, d.Carrier = "", "bbbb2222", "novox/app@bbbb (merged as bbbb2222, walk plan-2)" + d.Walk = &WalkSeen{ID: "plan-2", State: walkDone} + } + case EvRewalk: + d.MergedAs, d.AnsweredBy = "aaaa1111", "bbbb2222" + d.Walk = &WalkSeen{ID: "plan-2", State: walkFailed} case EvLifted: d.MergedAs, d.Lifted = "aaaa1111", "its group's composed check passed" d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} @@ -112,7 +120,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead, EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop, - EvHistory, EvRetire, EvCovered, EvLifted} + EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { @@ -150,8 +158,10 @@ func TestTheTableKeepsItsRules(t *testing.T) { } // The trunk rule: only a ready delivery is published, by its merge — or a held one published again when // what held it is gone (issue 316); nothing off the trunk delivers. + // And a merge whose walk carried a later commit for it and failed waits there for a walk of its own (ADR 0276). if r.To == Published && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) && - !(r.Event == EvLifted && len(r.From) == 1 && r.From[0] == Held) { + !(r.Event == EvLifted && len(r.From) == 1 && r.From[0] == Held) && + !(r.Event == EvRewalk && len(r.From) == 1 && r.From[0] == Delivering) { t.Errorf("%v —%s→ published: only a ready delivery's merge publishes", r.From, r.Event) } // Nothing a person did not decide ends a delivery as stopped or failed by observation from held. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index 327f881c..4523661e 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -33,10 +33,23 @@ func (h *Holder) waitsFor(d *Delivery) string { case Proposed: return "its check" case Ready: + if b, batched := h.batchOf(d); batched { + return batchSaid(b, h.Now()) + } + if d.MergedAs != "" { + return "a walk that answers its merge" + } return "its merge" case Rejected: return "a new head, or a recheck" case Published: + if d.waitsForItsOwnWalk() { + return "a walk of its own commit: its walk " + d.Walk.ID + ", which carried " + shortOf(d.AnsweredBy, 8) + + " for it, failed" + } + if d.Walk != nil && d.Walk.TakenOverBy != "" { + return "the walk " + d.Walk.TakenOverBy + ", which took its walk " + d.Walk.ID + " over" + } if d.Group != "" { return "its turn in group " + d.Group } @@ -154,12 +167,14 @@ func (h *Holder) Groups(all bool) []GroupLine { type StalledLine struct { ID string `json:"id"` // Number is the pull request's, for a head the forge never announced: the condition names it (novox/hq issue 347). - Number int `json:"number,omitempty"` - State State `json:"state"` - For string `json:"for"` - Bound string `json:"bound"` - H2 string `json:"h2"` - Says string `json:"says"` + Number int `json:"number,omitempty"` + State State `json:"state"` + // Walk is the walk a line of the state Unheld is about (novox/hq issue 362). + Walk string `json:"walk,omitempty"` + For string `json:"for"` + Bound string `json:"bound"` + H2 string `json:"h2"` + Says string `json:"says"` } // Stalled is the `stalled` verb: what the controller's self-check raises as `delivery..stalled`. @@ -172,7 +187,7 @@ func (h *Holder) Stalled() []StalledLine { if !pastBound(d, now) { continue } - b := Bounds[d.State] + b, _ := boundOf(d) h2 := "none: the state is the operator's" switch { case d.State == Proposed && d.CheckAsk == nil: @@ -192,6 +207,7 @@ func (h *Holder) Stalled() []StalledLine { Bound: b.For.String(), H2: h2, Says: b.Says}) } out = append(out, h.uncheckedLines(now)...) + out = append(out, h.orphanLines(now)...) sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out } @@ -355,7 +371,7 @@ func (h *Holder) Close(id, why string) (string, error) { h.mu.Unlock() return "", fmt.Errorf("no delivery %q", id) } - b, bounded := Bounds[d.State] + b, bounded := boundOf(d) if !bounded || len(b.H2) == 0 { h.mu.Unlock() return "", fmt.Errorf("%s is %s: H2 has no transition there — the state is the operator's", id, d.State)