mesh-delivery: a walk answers every merge it carries, never a delivery that ended (hq issue 362, ADR 0276)
mesh/merge-gate pass: builds mesh-delivery → novox; no bus step; every machine composes with the change as it did without (4 of 4 compose)
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery delivered

Two catalogue merges 18 s apart were walked by one walk at the later commit,
which superseded the first; this owner tied that walk to the dunst delivery
the supersession had ended and to no other, so nothing gave it its word and
claude-code was held for a person with a reason naming neither cause.

A delivery now follows the newest walk that answers its merge: one naming it
(delivery.merges, the controller's batches), its own commit, the walk that
took its own over (taken_over_by, or the superseded note), or, from a
controller that names no merges, an open walk of its branch made for a later
merge, moving all it moves, whose commit the forge says contains it. Only its
own walk's record moves an ended delivery. The word is asked once per walk,
when every delivery it carries is at its turn, a group's members on one walk
together. An earlier merge carried by a later commit ends covered with the
walk, and waits for a walk of its own when it fails. A merge no walk answers
is held with the true reason and listed at once; a waiting walk no live
delivery holds is listed as unheld; a merge in an assembling or queued batch
is shown, not held. Replay R362 fails before.
This commit is contained in:
jochen
2026-10-10 13:05:50 +02:00
parent 52f34d362a
commit 13ffae7706
11 changed files with 1370 additions and 103 deletions
+2
View File
@@ -7,7 +7,9 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
modules/messenger/cmd/messenger/messenger
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
modules/claude-code/cmd/claude-code/claude-code
modules/mesh-delivery/cmd/mesh-delivery/mesh-delivery
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
modules/messenger/messenger
modules/claude-code/claude-code
modules/mesh-watcher/mesh-watcher
modules/mesh-delivery/mesh-delivery
@@ -0,0 +1,414 @@
package main
import (
"fmt"
"sort"
"strings"
"time"
)
// Which merges a walk answers (novox/hq issue 362, ADR 0276).
//
// On 2026-10-10 two merges of the catalogue 18 seconds apart were walked by one walk at the later merge's
// commit, which superseded the first. This owner found the walk a delivery follows by the walk's id or by the
// commit it was merged as, final or not: the surviving walk matched the dunst delivery, which the supersession
// had already ended, so nothing gave it its word, and the claude-code delivery, merged as another commit,
// matched no walk and was held for a person after ten minutes, for a reason that named neither.
//
// A walk answers every merge it carries: the merges it names (`delivery.merges`, said by a controller that
// batches merges), the merge of its own commit, a merge whose walk it took over, and a merge of its repository
// and branch the forge says its commit contains. A delivery follows the newest walk that answers it, and only a
// delivery not yet ended is ever tied to a walk by what it answers: an ended delivery is moved only by the
// record of its own walk, which no longer moves it.
// seenWalk is a walk's last record and when it was heard.
type seenWalk struct {
walk Walk
at time.Time
}
// keptWalkFor is how long a walk no longer open is kept for a merge heard after it ended; a batch is kept as
// long, past its window's latest close.
const keptWalkFor = 10 * time.Minute
// answer is a delivery a walk answers and the commit of its repository the walk carries for it.
type answer struct {
d *Delivery
carried string
}
// walkAnswers is whether a walk answers a delivery's merge, and the commit it carries for it.
func walkAnswers(w Walk, d *Delivery) (string, bool) {
if w.Release != nil || w.IsBatch() || d.MergedAs == "" {
return "", false
}
if c, ok := w.carriedFor(d.Repository, d.MergedAs); ok {
return c, true
}
if d.Walk != nil && d.Walk.TakenOverBy == w.ID && namesNoMerges(w) && strings.EqualFold(w.Repository, d.Repository) &&
sameBranch(w.Branch, d.Base) {
// The walk that took its walk over, from a controller that names no merges: it answers every merge the one
// it took over answered, by its commit of the same repository and branch. A walk that names its merges
// answers only those (carriedFor, above).
return w.Commit, true
}
return "", false
}
// containedAnswer is, for a controller that does not name the merges a walk answers, whether an open walk
// answers a ready merged delivery by containment: of its repository and branch, made for a merge after it, the
// forge says its commit contains this merge, and it moves every module this merge's plan moves. A later walk that
// moves only what its own merge moved does not deliver this one, however much of the trunk it contains.
func (h *Holder) containedAnswer(w Walk, d *Delivery) (string, bool) {
if !h.mayContain(w, d) || !d.ContainedIn[w.Commit] {
return "", false
}
return w.Commit, true
}
// mayContain is whether an open walk is one to ask the forge about for a ready merged delivery (containedAnswer).
func (h *Holder) mayContain(w Walk, d *Delivery) bool {
if w.Release != nil || w.IsBatch() || !w.Open() || !namesNoMerges(w) || d.State != Ready || d.MergedAs == "" || d.Walk != nil ||
d.Plan == nil || len(d.Plan.Moved) == 0 || !strings.EqualFold(w.Repository, d.Repository) ||
!sameBranch(w.Branch, d.Base) || sameCommit(w.Commit, d.MergedAs) {
return false
}
for _, m := range d.Plan.Moved {
if _, moves := w.Modules[m]; !moves {
return false
}
}
// Made for a merge after this one: the walk's own merge, where this owner holds it, or else the walk itself.
after := w.Created
for _, o := range h.deliveries {
if strings.EqualFold(o.Repository, w.Repository) && sameCommit(o.MergedAs, w.Commit) && !o.MergedAt.IsZero() {
after = o.MergedAt
}
}
return after.After(d.MergedAt)
}
// namesNoMerges is whether a walk is from a controller that does not name the merges it answers.
func namesNoMerges(w Walk) bool { return w.Delivery == nil || len(w.Delivery.Merges) == 0 }
func sameBranch(a, b string) bool { return a == "" || b == "" || strings.EqualFold(a, b) }
// follows is whether a delivery takes a walk that answers it as the one it follows: it has none, it is that walk,
// that walk took its own over, or that walk is newer than the one it follows. A walk record kept before its
// creation was read is followed past only once it is superseded or failed.
func follows(d *Delivery, w Walk) bool {
switch {
case d.Walk == nil || d.Walk.ID == w.ID || d.Walk.TakenOverBy == w.ID:
return true
case d.Walk.Created.IsZero():
return d.Walk.State == walkSuperseded || d.Walk.State == walkFailed
}
return w.Created.After(d.Walk.Created)
}
// deliveriesWalking are the deliveries a walk record moves: every delivery that already follows it — ended or
// not, its own record — and every delivery not ended that it answers and that follows it now. In id order.
func (h *Holder) deliveriesWalking(w Walk) []answer {
var out []answer
for _, d := range h.deliveries {
if d.Walk != nil && d.Walk.ID == w.ID {
carried := d.AnsweredBy
if c, ok := walkAnswers(w, d); ok {
carried = c
}
out = append(out, answer{d: d, carried: carried})
continue
}
if d.State.Final() || !follows(d, w) {
continue
}
if c, ok := walkAnswers(w, d); ok {
out = append(out, answer{d: d, carried: c})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].d.ID < out[j].d.ID })
return out
}
// accounted is whether any delivery, ended or not, follows the walk or is answered by it: a walk known for
// what it answers is never made a delivery of its own.
func (h *Holder) accounted(w Walk) bool {
for _, d := range h.deliveries {
if d.Walk != nil && d.Walk.ID == w.ID {
return true
}
if _, ok := walkAnswers(w, d); ok {
return true
}
}
return false
}
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one.
func (h *Holder) heldWalk(w Walk) bool {
for _, d := range h.deliveries {
if d.State.Final() {
continue
}
if d.Walk != nil && d.Walk.ID == w.ID {
return true
}
if _, ok := walkAnswers(w, d); ok {
return true
}
}
return false
}
// pickWalk is the newest walk heard that answers a merged delivery with no walk yet; nil for none.
func (h *Holder) pickWalk(d *Delivery) (Walk, string, bool) {
var best Walk
var carried string
found := false
for _, s := range h.walks {
if !follows(d, s.walk) {
continue
}
c, ok := walkAnswers(s.walk, d)
if !ok {
c, ok = h.containedAnswer(s.walk, d)
}
if !ok {
continue
}
if !found || s.walk.Created.After(best.Created) || s.walk.Created.Equal(best.Created) && s.walk.ID > best.ID {
best, carried, found = s.walk, c, true
}
}
return best, carried, found
}
// linkAnswering ties a merged delivery with no walk to the newest walk heard that answers it; under the lock.
func (h *Holder) linkAnswering(d *Delivery) bool {
if d.State.Final() || d.MergedAs == "" || d.Walk != nil {
return false
}
w, carried, ok := h.pickWalk(d)
if !ok {
return false
}
h.linkWalk(d, w, carried)
delete(h.unmatched, w.ID)
return true
}
// containsAsks are the forge's asks whether an open walk of a merged delivery's repository and branch, at
// another commit, contains its merge: for a controller that does not name the merges a walk answers (issue 362),
// asked once per pair, the answer kept on the delivery.
func (h *Holder) containsAsks(d *Delivery, now time.Time) []func() {
var asks []func()
for _, s := range h.walks {
w := s.walk
if !h.mayContain(w, d) {
continue
}
if _, known := d.ContainedIn[w.Commit]; known {
continue
}
if a := h.askContains(d, w.Commit, now); a != nil {
asks = append(asks, a)
}
}
return asks
}
// batchOf is the batch a merged delivery's merge is in, when the controller said one; false for none.
func (h *Holder) batchOf(d *Delivery) (Walk, bool) {
if d.MergedAs == "" {
return Walk{}, false
}
for _, s := range h.walks {
if !s.walk.IsBatch() || s.walk.Delivery == nil {
continue
}
for _, m := range s.walk.Delivery.Merges {
if strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) {
return s.walk, true
}
}
}
return Walk{}, false
}
// batchSaid is what a batch holding a delivery's merge waits for, in words.
func batchSaid(b Walk, now time.Time) string {
said := "its batch " + b.ID + ": " + b.State
bt := b.Delivery.Batch
switch {
case b.State == walkQueued && bt != nil && bt.Behind != "":
said += " behind " + bt.Behind
case b.State == walkAssembling && bt != nil && bt.ClosesAt != nil:
left := bt.ClosesAt.Sub(now).Round(time.Second)
if left < 0 {
left = 0
}
said += fmt.Sprintf(", %s left", left)
if bt.AtMost != nil {
said += " (at the latest " + bt.AtMost.Local().Format("15:04:05") + ")"
}
}
return said + "; plan not yet calculated"
}
// unwalkedWhy is why a merged delivery no walk answers is held: the open walks of its repository and that none
// answers its merge, or that the controller holds none — what happened, not what might have.
func (h *Holder) unwalkedWhy(d *Delivery) string {
var open []string
for _, s := range h.walks {
w := s.walk
if w.Release == nil && !w.IsBatch() && w.Open() && strings.EqualFold(w.Repository, d.Repository) {
open = append(open, fmt.Sprintf("%s at %s", w.ID, shortOf(w.Commit, 8)))
}
}
sort.Strings(open)
head := fmt.Sprintf("merged as %s, and no walk answers its merge within %s: ", shortOf(d.MergedAs, 8), unwalked)
if len(open) == 0 {
return head + "the controller holds no open walk of " + d.Repository + " — its merge was not heard, or " +
"nothing the mesh holds follows " + orNone(d.Base) + ": a person decides"
}
return head + "the open walk(s) of " + d.Repository + " — " + strings.Join(open, ", ") + " — name no merge of " +
"it and the forge did not say they contain it: a person decides"
}
func orNone(s string) string {
if s == "" {
return "its branch"
}
return s
}
// orphanLines are the waiting walks no live delivery holds, past the grace a merge is given to be heard, as
// stalled lines in the state Unheld: nothing would ever give such a walk its word (issue 362). Under the lock.
func (h *Holder) orphanLines(now time.Time) []StalledLine {
var out []StalledLine
for id, first := range h.orphans {
s, known := h.walks[id]
if !known || now.Sub(first) < matchGrace {
continue
}
w := s.walk
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
}
return out
}
// Unheld is a waiting walk no live delivery holds: not a state of the table — only what a stalled line says.
const Unheld State = "unheld"
// seeOrphans keeps when each waiting walk no live delivery holds was first seen so, and forgets the rest; a
// walk still unmatched waits for its merge or its adoption instead. Under the lock.
func (h *Holder) seeOrphans(now time.Time) {
for id, s := range h.walks {
w := s.walk
_, unmatched := h.unmatched[id]
if unmatched || w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits || h.heldWalk(w) {
delete(h.orphans, id)
continue
}
if _, was := h.orphans[id]; !was {
h.orphans[id] = now
h.Logf("[mesh-delivery] the walk %s waits for this owner's word and no live delivery holds it: said once "+
"it is %s so", id, matchGrace)
}
}
for id := range h.orphans {
if _, known := h.walks[id]; !known {
delete(h.orphans, id)
}
}
}
// forgetWalks drops a walk record no longer open after keptWalkFor, and a batch heard of that long past its
// window's latest close. Under the lock.
func (h *Holder) forgetWalks(now time.Time) {
for id, s := range h.walks {
w := s.walk
switch {
case w.IsBatch():
end := s.at
if w.Delivery != nil && w.Delivery.Batch != nil && w.Delivery.Batch.AtMost != nil && w.Delivery.Batch.AtMost.After(end) {
end = *w.Delivery.Batch.AtMost
}
if now.Sub(end) > 3*time.Hour {
delete(h.walks, id)
}
case !w.Open() && now.Sub(s.at) > keptWalkFor:
delete(h.walks, id)
}
}
}
// --- the word, once per walk ------------------------------------------------------------------------------
// turnWant is a walk whose turn came for some of the deliveries it carries, and why.
type turnWant struct {
ids []string
whys []string
}
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
// Under the lock.
func (h *Holder) words() []func() {
walks := make([]string, 0, len(h.turning))
for walk := range h.turning {
walks = append(walks, walk)
}
sort.Strings(walks)
var asks []func()
for _, walk := range walks {
want := h.turning[walk]
var waiting []string
for _, d := range h.deliveries {
if d.State.Final() || d.Walk == nil || d.Walk.ID != walk || containsString(want.ids, d.ID) {
continue
}
waiting = append(waiting, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
if len(waiting) > 0 {
sort.Strings(waiting)
h.Logf("[mesh-delivery] the walk %s is not let go yet: %s carried by it not at its turn", walk,
strings.Join(waiting, ", "))
continue
}
ids, why := append([]string(nil), want.ids...), strings.Join(want.whys, "; ")
if h.asking[walk] {
continue // its word is being asked
}
h.asking[walk] = true
walk := walk
asks = append(asks, func() {
err := h.Controller.Deliver(walk, why)
h.mu.Lock()
defer h.mu.Unlock()
delete(h.asking, walk)
if err != nil {
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
return
}
for _, id := range ids {
d := h.deliveries[id]
if d == nil {
continue
}
d.LetGo, d.LetGoWhy = h.Now(), why
if d.Walk != nil && d.Walk.ID == walk {
d.Walk.Waits, d.Walk.GoBy = false, byOwner
}
h.settleAndKeep(d)
}
})
}
h.turning = nil
return asks
}
@@ -108,7 +108,11 @@ func statusOf(d *Delivery) (string, string) {
case Retired:
return "success", "history: merged before this owner delivered anything, never walked"
case Covered:
return "success", clip("covered: a later merge that contains it was delivered: "+d.CoveredBy, 140)
by := d.CoveredBy
if by == "" {
by = d.Carrier
}
return "success", clip("covered: a later merge that contains it was delivered: "+by, 140)
}
return "pending", string(d.State)
}
+181 -75
View File
@@ -45,6 +45,13 @@ type Holder struct {
refusals []string
// unchecked are the open pull requests' heads seen without a merge check (unchecked.go).
unchecked map[string]unchecked
// walks are the walks' and batches' last records (answers.go); orphans the waiting walks no live delivery
// holds, by when that was first seen; turning the walks whose turn came this tick, for some deliveries.
walks map[string]seenWalk
orphans map[string]time.Time
turning map[string]*turnWant
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
asking map[string]bool
}
// unmatched is a walk no delivery is known for yet: its merge not heard, or merged before this owner held.
@@ -86,6 +93,9 @@ func (h *Holder) init() {
h.askTried = map[string]time.Time{}
h.walkAsked = map[string]time.Time{}
h.unchecked = map[string]unchecked{}
h.walks = map[string]seenWalk{}
h.orphans = map[string]time.Time{}
h.asking = map[string]bool{}
h.started = h.Now()
}
@@ -401,13 +411,8 @@ func (h *Holder) PullMerged(p PullEvent) {
} else {
d.MergedAt = now
}
// A walk already heard of for this commit.
for id, u := range h.unmatched {
if strings.EqualFold(u.walk.Repository, d.Repository) && sameCommit(u.walk.Commit, d.MergedAs) {
h.linkWalk(d, u.walk)
delete(h.unmatched, id)
}
}
// A walk already heard of that answers this merge: its own, or one that names it.
h.linkAnswering(d)
// A merge long past heard only now, from the forge's word — replayed from the bus's history when this owner
// first held the seat — is history: it reached the machines before this owner was there to deliver it, and
// holding it for a person asks a person about the past (novox/hq issue 313).
@@ -417,7 +422,7 @@ func (h *Holder) PullMerged(p PullEvent) {
}
ts = append(ts, settle(d, Facts{Now: now})...)
if d.State == Held && d.HeldWhy == "" {
d.HeldWhy = "merged without a passing check: only a person decides that it goes on"
d.HeldWhy, d.Unwalked = "merged without a passing check: only a person decides that it goes on", false
}
h.moved(d, ts)
h.closeGroupOf(d)
@@ -562,8 +567,17 @@ func (h *Holder) walkMoved(w Walk) {
h.creditBacklog(w)
return
}
d := h.deliveryWalking(w)
if d == nil {
if old, seen := h.walks[w.ID]; !seen || w.Revision >= old.walk.Revision {
h.walks[w.ID] = seenWalk{walk: w, at: h.Now()}
}
if w.IsBatch() {
return // a batch is shown, never linked, adopted or let go (ADR 0276)
}
links := h.deliveriesWalking(w)
if len(links) == 0 {
if h.accounted(w) {
return // it answers only deliveries that ended or follow a newer walk
}
if _, seen := h.unmatched[w.ID]; !seen {
h.unmatched[w.ID] = unmatched{walk: w, first: h.Now()}
} else {
@@ -575,43 +589,72 @@ func (h *Holder) walkMoved(w Walk) {
}
return
}
if d.Walk != nil && d.Walk.ID == w.ID && w.Revision < d.Walk.Revision {
return // an older record arriving late
delete(h.unmatched, w.ID)
moved := map[string]bool{}
for _, a := range links {
d := a.d
if d.Walk != nil && d.Walk.ID == w.ID && w.Revision < d.Walk.Revision {
continue // an older record arriving late
}
h.linkWalk(d, w, a.carried)
h.settleAndKeep(d)
moved[d.ID] = true
}
h.linkWalk(d, w)
h.settleAndKeep(d)
// What its sends carried of other deliveries' builds.
for _, m := range w.Modules {
if m != nil && m.Gate != nil {
h.creditCarried(m.Gate, w.ID, d.ID)
h.creditCarried(m.Gate, w.ID, moved)
}
}
}
// deliveryWalking is the delivery a walk walks: the one that landed as its commit, or that it walked before.
func (h *Holder) deliveryWalking(w Walk) *Delivery {
for _, d := range h.deliveries {
if d.Walk != nil && d.Walk.ID == w.ID {
return d
}
// linkWalk records a walk on its delivery: what it is at, the commit it carries for it, and the machine steps
// its record says — each moved only as the step table allows. A delivery that follows a new walk takes its
// steps as that walk's, a new journey; and one held because no walk answered it is lifted.
func (h *Holder) linkWalk(d *Delivery, w Walk, carried string) {
switching := d.Walk != nil && d.Walk.ID != w.ID
if switching {
h.Logf("[mesh-delivery] %s follows the walk %s now (it followed %s, %s)", d.ID, w.ID, d.Walk.ID, d.Walk.State)
d.LetGo, d.LetGoWhy = time.Time{}, "" // a new walk waits for a word of its own
}
for _, d := range h.deliveries {
if strings.EqualFold(d.Repository, w.Repository) && sameCommit(d.MergedAs, w.Commit) {
return d
}
}
return nil
}
// linkWalk records a walk on its delivery: what it is at, and the machine steps its record says — each
// moved only as the step table allows.
func (h *Holder) linkWalk(d *Delivery, w Walk) {
d.Walk = w.Seen()
if carried != "" {
d.AnsweredBy = carried
}
d.Carrier = ""
if d.carriedLater() {
d.Carrier = fmt.Sprintf("merged as %s, carried by the walk %s", shortOf(d.AnsweredBy, 12), w.ID)
for _, o := range h.deliveries {
if o.ID != d.ID && strings.EqualFold(o.Repository, d.Repository) && sameCommit(o.MergedAs, d.AnsweredBy) {
d.Carrier = fmt.Sprintf("%s (merged as %s into %s, walk %s)", o.ID, shortOf(d.AnsweredBy, 8), orNone(d.Base), w.ID)
}
}
}
if d.State == Held && d.Unwalked && w.Open() {
d.HeldWhy = ""
d.Lifted = fmt.Sprintf("the walk %s, which answers its merge, opened", w.ID)
}
d.Unwalked = false // a walk answers it
for _, s := range StepsOf(w) {
if switching {
h.restep(d, s)
continue
}
h.step(d, s)
}
}
// restep puts a step of a walk a delivery newly follows, whatever the walk before said of that machine.
func (h *Holder) restep(d *Delivery, s Step) {
for i, old := range d.Steps {
if old.Machine == s.Machine && old.Module == s.Module {
d.Steps[i] = s
return
}
}
d.Steps = append(d.Steps, s)
}
// step moves one machine's step, or refuses a move the step table does not hold; it says whether it moved.
func (h *Holder) step(d *Delivery, s Step) bool {
for i, old := range d.Steps {
@@ -647,7 +690,7 @@ func (h *Holder) creditBacklog(w Walk) {
}
}
if w.Release.Gate != nil {
h.creditCarried(w.Release.Gate, w.ID, "")
h.creditCarried(w.Release.Gate, w.ID, nil)
}
if w.State == walkDone {
h.creditReleased(w)
@@ -704,7 +747,7 @@ func (h *Holder) askWalk(id string, now time.Time) func() {
}
// creditCarried gives the delivery that published each carried build its machine step.
func (h *Holder) creditCarried(g *WalkGate, walk, except string) {
func (h *Holder) creditCarried(g *WalkGate, walk string, except map[string]bool) {
state := StepJudging
switch {
case g.Rollback == "rolled-back":
@@ -716,7 +759,7 @@ func (h *Holder) creditCarried(g *WalkGate, walk, except string) {
}
for _, c := range g.Carried {
for _, d := range h.deliveries {
if d.ID == except || d.MergedAs == "" || !sameCommit(d.MergedAs, c.To) {
if except[d.ID] || d.MergedAs == "" || !sameCommit(d.MergedAs, c.To) {
continue
}
if h.step(d, Step{Machine: c.Node, Module: c.Module, Build: c.Build, State: state, Since: h.Now(),
@@ -763,11 +806,19 @@ func (h *Holder) plan() []func() {
now := h.Now()
var asks []func()
h.forgetWalks(now)
// Walks no merge was heard for: made deliveries of their own once their merge had time to be heard.
for id, u := range h.unmatched {
if d := h.deliveryWalking(u.walk); d != nil {
h.linkWalk(d, u.walk)
h.settleAndKeep(d)
if links := h.deliveriesWalking(u.walk); len(links) > 0 {
for _, a := range links {
h.linkWalk(a.d, u.walk, a.carried)
h.settleAndKeep(a.d)
}
delete(h.unmatched, id)
continue
}
if h.accounted(u.walk) {
delete(h.unmatched, id)
continue
}
@@ -813,6 +864,39 @@ func (h *Holder) plan() []func() {
}
}
// Merged with no walk yet: tied to the newest walk heard that answers it — by name, by its own commit, or by
// the forge's word that the walk's commit contains it, asked here (novox/hq issue 362).
for _, d := range h.deliveries {
if h.linkAnswering(d) {
h.settleAndKeep(d)
continue
}
asks = append(asks, h.containsAsks(d, now)...)
}
// Following a walk that was taken over: the walk that took it over, read as soon as it is heard of, asked for
// by its id when it is not.
for _, d := range h.deliveries {
if d.State.Final() || d.Walk == nil || d.Walk.TakenOverBy == "" {
continue
}
if s, known := h.walks[d.Walk.TakenOverBy]; known {
if c, ok := walkAnswers(s.walk, d); ok {
h.linkWalk(d, s.walk, c)
} else {
// The walk that took its own over does not answer it: superseded, as before ADR 0276.
h.Logf("[mesh-delivery] %s: the walk %s, which took %s over, does not answer its merge", d.ID,
d.Walk.TakenOverBy, d.Walk.ID)
d.Walk.TakenOverBy = ""
}
h.settleAndKeep(d)
continue
}
if a := h.askWalk(d.Walk.TakenOverBy, now); a != nil {
asks = append(asks, a)
}
}
// Held, or merged with no walk opened for it: ended by what is already true of it, never asked of a person
// (novox/hq issue 319). A plan that moves nothing is delivered; one whose every module a later merge into the
// same trunk delivered is covered. Read again each tick, so a delivery read back after a restart, or held
@@ -853,13 +937,13 @@ func (h *Holder) plan() []func() {
// Merged, and no walk opened for it after the walks were read again: never silent — held, saying so.
for _, d := range h.deliveries {
if h.unwalkedPast(d, now) {
d.HeldWhy = "merged, and the controller opened no walk for it within " + unwalked.String() + " — nothing it " +
"holds follows that branch, or the merge was not heard: a person decides"
d.HeldWhy, d.Unwalked = h.unwalkedWhy(d), true
h.settleAndKeep(d)
}
}
// Groups: membership, order, composed check, and their turns.
h.turning = map[string]*turnWant{}
grouped := map[string]bool{}
for _, g := range h.groups {
members := h.membersOf(g)
@@ -874,10 +958,13 @@ func (h *Holder) plan() []func() {
if grouped[d.ID] || d.Group != "" && h.groups[d.Group] != nil {
continue
}
if a := h.turn(d, "its turn: a delivery in no group"); a != nil {
asks = append(asks, a)
}
h.turn(d, "its turn: a delivery in no group")
}
// The word, once per walk, when every live delivery it carries is at its turn.
asks = append(asks, h.words()...)
// Waiting walks no live delivery holds: listed by `stalled` (issue 362).
h.seeOrphans(now)
// Final deliveries past their keeping, once nothing is owed for them.
for id, d := range h.deliveries {
@@ -893,6 +980,9 @@ func (h *Holder) plan() []func() {
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
// again after it: what holds it for a person, unless something already ends it.
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
if _, batched := h.batchOf(d); batched {
return false // its batch is assembling or queued: a walk is not owed yet (ADR 0276)
}
return d.State == Ready && d.MergedAs != "" && d.Walk == nil && !d.MovesNothing() && d.HeldWhy == "" &&
now.Sub(d.MergedAt) > unwalked && h.reconciled.After(d.MergedAt.Add(unwalked))
}
@@ -1075,11 +1165,18 @@ func (h *Holder) adopt(w Walk) {
d := &Delivery{ID: IDOf(w.Repository, w.Commit), Repository: strings.ToLower(w.Repository), Commit: w.Commit,
Base: w.Branch, MergedAs: w.Commit, Created: now}
if old := h.deliveries[d.ID]; old != nil {
h.linkWalk(old, w)
if old.State.Final() {
// Never revived: an ended delivery is not the walk's; the walk, held by nothing, is listed by
// `stalled` (issue 362).
h.Logf("[mesh-delivery] the walk %s is at %s, whose delivery has ended (%s): not adopted by it", w.ID,
shortOf(w.Commit, 12), old.State)
return
}
h.linkWalk(old, w, w.Commit)
h.settleAndKeep(old)
return
}
h.linkWalk(d, w)
h.linkWalk(d, w, w.Commit)
var t Transition
var err error
if d.Walk.Waits {
@@ -1098,7 +1195,9 @@ func (h *Holder) adopt(w Walk) {
h.settleAndKeep(d)
}
// turn is the ask that lets a published delivery's waiting walk start; nil when it is not its turn.
// turn says a published delivery's waiting walk may start as far as that delivery goes; nil when it is not its
// turn. The word itself is asked once per walk by words, when every live delivery the walk carries is at its turn
// (ADR 0276 decision 5). What it answers is only whether this delivery's turn came.
func (h *Holder) turn(d *Delivery, why string) func() {
if d.Walk == nil || !d.Walk.Waits || !d.LetGo.IsZero() {
return nil
@@ -1106,25 +1205,22 @@ func (h *Holder) turn(d *Delivery, why string) func() {
if d.State != Published && d.State != Delivering {
return nil
}
id, walk := d.ID, d.Walk.ID
return func() {
err := h.Controller.Deliver(walk, why)
h.mu.Lock()
defer h.mu.Unlock()
d := h.deliveries[id]
if d == nil {
return
}
if err != nil {
h.Logf("[mesh-delivery] %s's walk %s could not be let go yet: %v", id, walk, err)
return
}
d.LetGo, d.LetGoWhy = h.Now(), why
if d.Walk != nil && d.Walk.ID == walk {
d.Walk.Waits, d.Walk.GoBy = false, byOwner
}
h.settleAndKeep(d)
if d.Walk.State != walkBuilding && d.Walk.State != walkRolling {
return nil // a walk superseded, done or failed is no walk to start
}
if h.turning == nil {
h.turning = map[string]*turnWant{}
}
want := h.turning[d.Walk.ID]
if want == nil {
want = &turnWant{}
h.turning[d.Walk.ID] = want
}
if !containsString(want.ids, d.ID) {
want.ids = append(want.ids, d.ID)
want.whys = append(want.whys, d.ID+": "+why)
}
return func() {}
}
// membersOf are a group's current members, by id order.
@@ -1263,13 +1359,20 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() {
// Composed for these very heads: a verdict on another set of heads is no verdict on these.
composed := g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") &&
sameMembers(g.Check.Commits, commitsOf(members))
// A member's turn registered, the members after it that the same walk carries take theirs too: one walk is let
// go once, for all of them (ADR 0276 decision 5). A member on another walk, or not at its turn, waits.
var asks []func()
sharing := ""
for i, m := range ordered {
d := byID[m.ID]
switch {
case d.State.Reached():
if d.State.Reached() {
continue
}
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
return asks
}
switch {
case d.State == Failed || d.State == Stopped || d.State == Superseded:
var asks []func()
for _, later := range ordered[i+1:] {
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
asks = append(asks, a)
@@ -1286,32 +1389,35 @@ func (h *Holder) groupTurns(g *Group, members []*Delivery) []func() {
h.settleAndKeep(d)
if d.State == Published {
if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil {
return []func(){a}
asks, sharing = append(asks, a), d.Walk.ID
continue
}
}
}
return nil
return asks
case d.State == Published:
// Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316).
if !composed && d.HeldWhy == "" && g.Check != nil && g.Check.Verdict == "" &&
sameMembers(g.Check.Commits, commitsOf(members)) {
return nil
return asks
}
if !composed && d.HeldWhy == "" {
d.HeldWhy = fmt.Sprintf("its group %s's composed check did not pass for the heads that merged; a person "+
"decides that it goes on", g.ID)
d.Unwalked = false
h.settleAndKeep(d)
return nil
return asks
}
if a := h.turn(d, fmt.Sprintf("its turn in group %s (%d of %d)", g.ID, i+1, len(ordered))); a != nil {
return []func(){a}
asks, sharing = append(asks, a), d.Walk.ID
continue
}
return nil
return asks
default:
return nil // not on the trunk yet, held, or delivering: the members after it wait
return asks // not on the trunk yet, held, or delivering: the members after it wait
}
}
return nil
return asks
}
// stopInGroup stops a member after one that failed, and its walk if it has one open.
@@ -162,7 +162,8 @@ func TestAFailedGateIsRolledBackAndTheDeliveryFails(t *testing.T) {
}
}
// A newer merge's walk took over this one's: superseded, from published or delivering.
// A walk superseded that names no walk taking it over (a record from before ADR 0276): superseded, from
// published or delivering. One that names its successor is followed (TestADeliveryFollowsTheWalkThatTookItsOver).
func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head)
@@ -172,7 +173,7 @@ func TestANewerWalkSupersedesAnOlderDelivery(t *testing.T) {
walk := aWalk("plan-1", "novox/app", merge, true)
w.h.WalkMoved(w.ctl.put(walk))
walk.State = walkSuperseded
walk.Note = "superseded at tier 0 by plan-2"
walk.Note = "superseded"
w.ctl.down = true // its word could not be given meanwhile
w.settleAll()
w.h.WalkMoved(w.ctl.put(walk))
@@ -0,0 +1,434 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 362, ADR 0276: two merges of one repository 18 seconds apart, walked by one walk at the later
// merge's commit, which superseded the first. Every delivery the walk answers follows it and ends with it; no
// walk is tied to a delivery that ended; and what cannot move is said at once.
// A controller that names the merges a walk answers (ADR 0276): the forge is not asked, and the earlier merge is
// tied at once.
func TestAWalkThatNamesItsMergesIsFollowedByEachOfThem(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
for _, id := range []string{IDOf(cat, headCC), IDOf(cat, headDu)} {
if d := w.delivery(id); d.State != Published || d.Walk == nil || d.Walk.ID != "plan-9" {
t.Fatalf("%s is %s on %v", id, d.State, d.Walk)
}
}
w.settleAll()
if len(w.forge.containsAsks) != 0 {
t.Fatalf("the forge was asked %v for a walk that names its merges", w.forge.containsAsks)
}
if len(w.ctl.delivers) != 1 {
t.Fatalf("the word was asked %v", w.ctl.delivers)
}
}
// A walk that failed marks no merge delivered: the merge it carried its own commit for fails, and the earlier
// merge it carried a later commit for waits for a walk of its own.
func TestAFailedWalkMarksNoMergeItCarriedDelivered(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk("plan-9")
walk.State, walk.Note = walkFailed, "dunst failed its gate on anchor"
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(IDOf(cat, headDu)); s != Failed {
t.Fatalf("dunst is %s", s)
}
cc := w.delivery(IDOf(cat, headCC))
if cc.State != Published || cc.State.Reached() {
t.Fatalf("claude-code is %s", cc.State)
}
// Its own walk, when the controller opens one, is followed.
own := catWalk("plan-10", mergeCC, atDu.Add(time.Hour), "claude-code")
w.h.WalkMoved(w.ctl.put(own))
w.settleAll()
w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-10"), w.now)))
w.settleAll()
if s := w.state(IDOf(cat, headCC)); s != Delivered {
t.Fatalf("claude-code on its own walk is %s", s)
}
}
// Never revived: a walk at the commit of a delivery that ended is not tied to it, not adopted, and not let go —
// it is listed by `stalled` as a walk no live delivery holds.
func TestAWalkIsNeverTiedToADeliveryThatEnded(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(supersededBy(first, "")))
// A record from before ADR 0276: superseded, naming no walk — the delivery ends superseded.
sup := w.ctl.walk("plan-1")
sup.Note = "superseded"
w.h.WalkMoved(w.ctl.put(sup))
idDu := IDOf(cat, headDu)
if s := w.state(idDu); s != Superseded {
t.Fatalf("dunst is %s", s)
}
orphan := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "dunst")
w.h.WalkMoved(w.ctl.put(orphan))
w.settleAll()
w.later(3 * time.Minute)
w.settleAll()
if d := w.delivery(idDu); d.State != Superseded || d.Walk.ID != "plan-1" {
t.Fatalf("the ended delivery is %s on %s", d.State, d.Walk.ID)
}
for _, asked := range w.ctl.delivers {
if asked == "plan-2" {
t.Fatal("a walk no live delivery holds was let go")
}
}
var found bool
for _, l := range w.h.Stalled() {
if l.State == Unheld && l.Walk == "plan-2" {
found = true
}
}
if !found {
t.Fatalf("stalled does not list the walk no delivery holds: %+v", w.h.Stalled())
}
}
// A merge no walk answers is held after ten minutes with the true reason, and listed by `stalled` at once; a walk
// that answers it later lifts it.
func TestAMergeNoWalkAnswersIsHeldTrulyAndSaidAtOnce(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
// An open walk of the repository that moves something else: it does not answer this merge.
w.h.WalkMoved(w.ctl.put(catWalk("plan-5", "eeeeeeeeeeee5555", atCC.Add(time.Minute), "dunst")))
w.later(11 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
id := IDOf(cat, headCC)
d := w.delivery(id)
if d.State != Held || !strings.Contains(d.HeldWhy, "plan-5") || !strings.Contains(d.HeldWhy, "name no merge") {
t.Fatalf("it is %s: %q", d.State, d.HeldWhy)
}
lines := w.h.Stalled()
if len(lines) != 1 || lines[0].ID != id || lines[0].Bound != "0s" {
t.Fatalf("stalled lists %+v", lines)
}
// The controller opens a walk that names it.
walk := catWalk("plan-6", mergeCC, w.now, "claude-code")
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(id); s != Delivering || len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-6" {
t.Fatalf("with its walk it is %s, let go %v", s, w.ctl.delivers)
}
}
// One word per walk, and none while a delivery it carries is not at its turn.
func TestAWalkIsLetGoOnlyWhenEveryDeliveryItCarriesIsAtItsTurn(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json")
w.h.PullUpdated(cc)
w.h.PullUpdated(du)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
// dunst merges unchecked: held for a person.
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.h.PullMerged(mergedAt(du, mergeDu, atDu))
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if s := w.state(IDOf(cat, headDu)); s != Held {
t.Fatalf("dunst is %s", s)
}
if len(w.ctl.delivers) != 0 {
t.Fatalf("the walk was let go %v while it carries a held delivery", w.ctl.delivers)
}
if _, err := w.h.Release(IDOf(cat, headDu), "checked by hand", "jochen"); err != nil {
t.Fatal(err)
}
w.settleAll()
if len(w.ctl.delivers) != 1 {
t.Fatalf("released, the word was asked %v", w.ctl.delivers)
}
}
// A merge in a batch still assembling is said so, is not held for a person, and nothing is let go.
func TestAMergeInAnAssemblingBatchIsShownAndNotHeld(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
closes, atMost := atCC.Add(90*time.Second), atCC.Add(10*time.Minute)
batch := Walk{ID: "plan-b1", Repository: cat, Branch: "main", Commit: mergeCC, Created: atCC, State: walkAssembling,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{{Repository: cat, Commit: mergeCC}},
Batch: &WalkBatch{ClosesAt: &closes, AtMost: &atMost}}}
w.h.WalkMoved(w.ctl.put(batch))
w.later(48 * time.Second)
w.settleAll()
id := IDOf(cat, headCC)
var line Line
for _, l := range w.h.Deliveries("", "", "", true) {
if l.ID == id {
line = l
}
}
if !strings.Contains(line.Waits, "plan-b1: assembling, 42s left") || !strings.Contains(line.Waits, "plan not yet calculated") {
t.Fatalf("it waits for %q", line.Waits)
}
// Queued behind a walk for longer than a walk opens within: still not held.
batch = w.ctl.walk("plan-b1")
batch.State, batch.Delivery.Batch.Behind = walkQueued, "plan-7"
w.h.WalkMoved(w.ctl.put(batch))
w.later(20 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
if d := w.delivery(id); d.State != Ready || d.Walk != nil {
t.Fatalf("a merge in a queued batch is %s on %v", d.State, d.Walk)
}
if len(w.ctl.delivers) != 0 {
t.Fatalf("a batch was let go: %v", w.ctl.delivers)
}
// Cut: the batch is a walk now, under its id.
batch.State, batch.Delivery.Batch = walkBuilding, nil
batch.Tiers, batch.Modules = [][]string{{"claude-code"}}, map[string]*WalkModule{"claude-code": {}}
w.h.WalkMoved(w.ctl.put(batch))
w.settleAll()
if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-b1" {
t.Fatalf("cut, it is %s on %v", d.State, d.Walk)
}
}
// An older record of the walk a delivery no longer follows, arriving late, does not take it back.
func TestALateRecordOfAnOlderWalkDoesNotTakeADeliveryBack(t *testing.T) {
w := twoMerges(t)
w.forge.contains[mergeCC+" in "+mergeDu] = true
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")))
w.h.WalkMoved(first) // the older record, again
if d := w.delivery(IDOf(cat, headDu)); d.Walk.ID != "plan-2" {
t.Fatalf("dunst went back to %s", d.Walk.ID)
}
}
// A walk no merge was heard for, at the commit of a delivery that ended, is not adopted by that delivery: it is
// listed by `stalled` as a walk no live delivery holds.
func TestAnUnheardWalkIsNotAdoptedByADeliveryThatEnded(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.PullClosed(cc)
id := IDOf(cat, headCC)
if s := w.state(id); s != Stopped {
t.Fatalf("closed unmerged it is %s", s)
}
w.h.WalkMoved(w.ctl.put(catWalk("plan-3", headCC, atCC.Add(time.Minute), "claude-code")))
w.later(3 * time.Minute)
w.settleAll()
w.later(3 * time.Minute)
w.settleAll()
if d := w.delivery(id); d.State != Stopped || d.Walk != nil {
t.Fatalf("the stopped delivery is %s on %v", d.State, d.Walk)
}
var found bool
for _, l := range w.h.Stalled() {
found = found || l.State == Unheld && l.Walk == "plan-3"
}
if !found || len(w.ctl.delivers) != 0 {
t.Fatalf("stalled lists %+v; let go %v", w.h.Stalled(), w.ctl.delivers)
}
}
// Two members of one delivery group carried by one walk: let go once, for both (review of #171: the group's turn
// stopped at its first member, and the walk waited for the second for ever).
func TestAWalkCarryingTwoMembersOfAGroupIsLetGoOnce(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
a := pr("novox/a", 1, "aaaaaaaaaaaa0001", "feat/y", "x.go")
b := pr("novox/b", 2, "bbbbbbbbbbbb0002", "feat/y", "y.go")
w.h.PullUpdated(a)
w.h.PullUpdated(b)
w.h.Checked(verdictMoving("novox/a", 1, "aaaaaaaaaaaa0001", "app-a"))
w.h.Checked(verdictMoving("novox/b", 2, "bbbbbbbbbbbb0002", "app-b"))
w.settleAll()
g := w.h.groups["feat/y"]
if g == nil || g.Check == nil {
t.Fatalf("the group is %+v", g)
}
w.h.Checked(CheckedEvent{Group: "feat/y", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"})
w.h.PullMerged(mergedAt(a, "aaaaaaaaaaaa1111", atCC))
w.h.PullMerged(mergedAt(b, "bbbbbbbbbbbb2222", atDu))
batch := Walk{ID: "plan-batch", Repository: "novox/a", Branch: "main", Commit: "aaaaaaaaaaaa1111",
Created: atDu.Add(time.Minute), State: walkBuilding, Tiers: [][]string{{"app-a"}, {"app-b"}},
Modules: map[string]*WalkModule{"app-a": {}, "app-b": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: "novox/a", Commit: "aaaaaaaaaaaa1111"}, {Repository: "novox/b", Commit: "bbbbbbbbbbbb2222"}}}}
w.h.WalkMoved(w.ctl.put(batch))
w.settleAll()
if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-batch" {
t.Fatalf("the word was asked %v, want once for plan-batch", got)
}
for _, id := range []string{IDOf("novox/a", "aaaaaaaaaaaa0001"), IDOf("novox/b", "bbbbbbbbbbbb0002")} {
if s := w.state(id); s != Delivering {
t.Fatalf("%s is %s", id, s)
}
}
}
// A walk that names its merges answers only those: a delivery whose walk it took over and which it does not name
// is not tied to it, and ends superseded once that is known.
func TestAWalkThatNamesItsMergesAnswersOnlyThose(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
other := catWalk("plan-2", "ffffffffffff9999", atDu.Add(6*time.Second), "dunst")
other.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: "ffffffffffff9999"}}
w.h.WalkMoved(w.ctl.put(other))
w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2")))
w.settleAll()
d := w.delivery(IDOf(cat, headDu))
if d.State != Superseded || d.Walk.ID != "plan-1" {
t.Fatalf("dunst is %s on %s", d.State, d.Walk.ID)
}
for _, asked := range w.ctl.delivers {
if asked == "plan-2" {
t.Fatal("a walk that does not name dunst was let go for it")
}
}
}
// The forge's containment ties a merge to a walk only when the walk moves everything the merge moves and was made
// for a later merge.
func TestContainmentAloneDoesNotTieAMerge(t *testing.T) {
for _, c := range []struct {
name string
modules []string
created time.Time
names bool
}{
{"the later walk moves only its own module", []string{"dunst"}, atDu.Add(5 * time.Second), false},
{"the walk was made for an earlier merge", []string{"claude-code", "dunst"}, atCC.Add(-time.Minute), false},
{"the walk names its merges, and not this one", []string{"claude-code", "dunst"}, atDu.Add(5 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
w.h.PullUpdated(cc)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.forge.contains[mergeCC+" in "+mergeDu] = true
walk := catWalk("plan-2", mergeDu, c.created, c.modules...)
if c.names {
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeDu}}
}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
if d := w.delivery(IDOf(cat, headCC)); d.Walk != nil || len(w.forge.containsAsks) != 0 {
t.Fatalf("it follows %v; the forge was asked %v", d.Walk, w.forge.containsAsks)
}
})
}
}
// After the walk that carried a later commit for it failed, a merge stays published — once — and is said within
// the ten minutes a walk opens within, naming what happened.
func TestAMergeWhoseCarryingWalkFailedWaitsOnceAndIsSaid(t *testing.T) {
w := twoMerges(t)
walk := catWalk("plan-9", mergeDu, atDu.Add(5*time.Second), "claude-code", "dunst")
walk.Delivery.Merges = []WalkMerge{{Repository: cat, Commit: mergeCC, Carried: mergeDu}, {Repository: cat, Commit: mergeDu}}
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
walk = w.ctl.walk("plan-9")
walk.State = walkFailed
w.h.WalkMoved(w.ctl.put(walk))
w.settleAll()
id := IDOf(cat, headCC)
d := w.delivery(id)
rewalks := 0
for _, tr := range d.Transitions {
if tr.Event == EvRewalk {
rewalks++
}
}
if d.State != Published || rewalks != 1 {
t.Fatalf("it is %s after %d rewalk(s)", d.State, rewalks)
}
w.later(11 * time.Minute)
var line *StalledLine
for _, l := range w.h.Stalled() {
if l.ID == id {
l := l
line = &l
}
}
if line == nil || !strings.Contains(line.Says, "no walk of its own commit") {
t.Fatalf("stalled says %+v", line)
}
}
// A delivery that ended is never tied to a walk after: not one that answers its merge, not the walk that took its
// own over.
func TestADeliveryThatEndedFollowsNoWalk(t *testing.T) {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atCC)
w.ctl.planned = planMoving(cat)
p := pr(cat, 103, "eeeeeeeeeeee0003", "docs/x", "README.md")
w.h.PullUpdated(p)
w.h.PullMerged(mergedAt(p, "eeeeeeeeeeee3333", atCC))
w.settleAll()
nothing := IDOf(cat, "eeeeeeeeeeee0003")
if s := w.state(nothing); s != Delivered {
t.Fatalf("a merge moving nothing is %s", s)
}
w.h.WalkMoved(w.ctl.put(catWalk("plan-4", "eeeeeeeeeeee3333", atCC.Add(time.Minute), "app")))
w.settleAll()
if d := w.delivery(nothing); d.Walk != nil {
t.Fatalf("a delivered merge was tied to %s", d.Walk.ID)
}
w2 := twoMerges(t)
first := w2.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w2.h.WalkMoved(first)
idDu := IDOf(cat, headDu)
if _, err := w2.h.Stop(idDu, "not now", "jochen"); err != nil {
t.Fatal(err)
}
w2.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst"))
w2.h.WalkMoved(w2.ctl.put(supersededBy(w2.ctl.walk("plan-1"), "plan-2")))
w2.settleAll()
if d := w2.delivery(idDu); d.State != Stopped || d.Walk.ID != "plan-1" {
t.Fatalf("the stopped delivery is %s on %s", d.State, d.Walk.ID)
}
}
// Today's controller, merges A then B minutes apart: B's walk took A's over before it started. A follows it and
// ends covered by B, rather than superseded (ADR 0276 decision 4).
func TestAnEarlierMergeFollowsTheWalkThatTookItsOver(t *testing.T) {
w := twoMerges(t)
first := w.ctl.put(catWalk("plan-1", mergeCC, atCC.Add(5*time.Second), "claude-code"))
w.h.WalkMoved(first)
w.h.WalkMoved(w.ctl.put(catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")))
w.h.WalkMoved(w.ctl.put(supersededBy(w.ctl.walk("plan-1"), "plan-2")))
w.settleAll()
idCC := IDOf(cat, headCC)
if d := w.delivery(idCC); d.Walk.ID != "plan-2" || len(w.ctl.delivers) != 1 {
t.Fatalf("claude-code follows %s; let go %v", d.Walk.ID, w.ctl.delivers)
}
w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now)))
w.settleAll()
if s := w.state(idCC); s != Covered {
t.Fatalf("claude-code is %s", s)
}
}
@@ -68,7 +68,14 @@ type Delivery struct {
// ContainedIn is the forge's answer, by a later merge commit, to whether that commit contains this one's
// merge (gitea_contains): asked once per pair, and kept (ADR 0270).
ContainedIn map[string]bool `json:"contained_in,omitempty"`
// Walk is the controller's walk of its trunk commit, as last read; Steps the machines it reached.
// AnsweredBy is the commit of its repository the walk it follows carries for it: its own merge, or a later merge
// of its branch that contains it (novox/hq ADR 0276). Unwalked says it is held because no walk answered it.
AnsweredBy string `json:"answered_by,omitempty"`
// Carrier says, for a merge a later one carries, which: its delivery and walk, in words.
Carrier string `json:"carrier,omitempty"`
Unwalked bool `json:"unwalked,omitempty"`
// Walk is the walk that delivers it, as last read: its own trunk commit's, or one that carries a later merge
// of its branch; Steps the machines it reached.
Walk *WalkSeen `json:"walk,omitempty"`
Steps []Step `json:"steps,omitempty"`
// LetGo is when this owner asked the controller to start its walk, and why.
@@ -166,6 +173,16 @@ type MachinePlan struct {
Waits []string `json:"waits,omitempty"`
}
// waitsForItsOwnWalk is whether it is published again after the walk that carried a later commit for it failed.
func (d *Delivery) waitsForItsOwnWalk() bool {
return d.State == Published && d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater()
}
// carriedLater is whether the walk it follows carries a later merge of its branch for it, not its own.
func (d *Delivery) carriedLater() bool {
return d.AnsweredBy != "" && d.MergedAs != "" && !sameCommit(d.AnsweredBy, d.MergedAs)
}
// MovesNothing is whether the delivery's plan moves no module the mesh holds: nothing for a walk to do. A
// module it adds is not moved: the controller builds and registers it and sends it nowhere, and opens no walk
// for it (novox/hq issues 300 and 319). A delivery with no plan known is not said to move nothing.
@@ -337,7 +354,7 @@ type WalkRelease struct {
By string `json:"by,omitempty"`
}
// WalkDelivery is what a walk waits for.
// WalkDelivery is what a walk waits for, and which merges it answers (novox/hq ADR 0276).
type WalkDelivery struct {
Awaits string `json:"awaits"`
Go *time.Time `json:"go,omitempty"`
@@ -345,6 +362,81 @@ type WalkDelivery struct {
Why string `json:"why,omitempty"`
Stopped string `json:"stopped,omitempty"`
StoppedWhy string `json:"stopped_why,omitempty"`
// Merges are every merge the walk answers: each merge commit, its repository, and the commit of that
// repository the walk carries — the merge itself, or a later merge of its branch that contains it. Absent
// from a controller that does not say it: the walk then answers the merge of its own commit, and a merge
// the forge says its commit contains.
Merges []WalkMerge `json:"merges,omitempty"`
// TakenOverBy is, on a walk folded before it started, the walk that took it over and answers every merge
// it answered.
TakenOverBy string `json:"taken_over_by,omitempty"`
// Batch is set while the record is a batch, not yet a walk: its state `assembling` or `queued`. A batch, once
// cut, is the walk under the same id: its next record is `building`. A merge this owner reads in a batch is
// not held as unwalked while the batch is heard of.
Batch *WalkBatch `json:"batch,omitempty"`
}
// WalkMerge is one merge a walk answers.
type WalkMerge struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Carried is the commit of the repository the walk carries for it: Commit itself, or a later merge of its
// branch that contains it. Empty means Commit.
Carried string `json:"carried,omitempty"`
}
// WalkBatch is a batch's window: when it closes unless another merge arrives, when it closes at the latest,
// and, once closed, the walk it waits behind.
type WalkBatch struct {
ClosesAt *time.Time `json:"closes_at,omitempty"`
AtMost *time.Time `json:"at_most,omitempty"`
Behind string `json:"behind,omitempty"`
}
// The states a batch's record is in before it is a walk (novox/hq ADR 0276).
const (
walkAssembling = "assembling"
walkQueued = "queued"
)
// IsBatch is whether the record is a batch still to be cut into a walk.
func (w Walk) IsBatch() bool { return w.State == walkAssembling || w.State == walkQueued }
// supersededNote reads the walk that took a superseded walk over from its note, for a controller that does not
// say TakenOverBy: "superseded at tier N by plan-… (…)".
var supersededNote = regexp.MustCompile(`^superseded at tier \d+ by (plan-[0-9A-Za-z_-]+)`)
// takenOverBy is the walk that took this one over, said or read from its note; empty for none.
func (w Walk) takenOverBy() string {
if w.State != walkSuperseded {
return ""
}
if w.Delivery != nil && w.Delivery.TakenOverBy != "" {
return w.Delivery.TakenOverBy
}
if m := supersededNote.FindStringSubmatch(w.Note); m != nil {
return m[1]
}
return ""
}
// carriedFor is the commit the walk carries for a merge of a repository, and whether it answers that merge by
// what it says: the merge named among its merges, or its own commit.
func (w Walk) carriedFor(repository, merge string) (string, bool) {
if w.Delivery != nil {
for _, m := range w.Delivery.Merges {
if strings.EqualFold(m.Repository, repository) && sameCommit(m.Commit, merge) {
if m.Carried != "" {
return m.Carried, true
}
return m.Commit, true
}
}
}
if strings.EqualFold(w.Repository, repository) && sameCommit(w.Commit, merge) {
return w.Commit, true
}
return "", false
}
// WalkSeen is what a delivery keeps of its walk.
@@ -363,6 +455,10 @@ type WalkSeen struct {
Updated time.Time `json:"updated"`
// Branch is the branch the walk's commit was merged into; empty in a record kept before it was read.
Branch string `json:"branch,omitempty"`
// Created is when the controller made the walk: a delivery follows the newest walk that answers it.
Created time.Time `json:"created"`
// TakenOverBy is the walk that took a superseded one over (novox/hq ADR 0276): its deliveries follow it.
TakenOverBy string `json:"taken_over_by,omitempty"`
}
// Seen is a walk as a delivery keeps it.
@@ -376,6 +472,8 @@ func (w Walk) Seen() *WalkSeen {
s.StoppedBy = d.Stopped
}
s.Branch = w.Branch
s.Created = w.Created
s.TakenOverBy = w.takenOverBy()
return s
}
@@ -0,0 +1,127 @@
package main
import (
"strings"
"testing"
"time"
)
// Replay R362 (novox/hq issue 362, mesh-lab's register): two merges of the catalogue 18 seconds apart, walked by
// one walk at the later merge's commit, which superseded the first — as the controller of 2026-10-10 said it, with
// no `merges` and the successor named only in the superseded walk's note. Written with only what mesh-delivery
// had before its fix (and fakes_test.go, covered_test.go's helpers), so it can be laid over the commit before.
const (
cat = "novox/mesh-catalog"
headCC = "cccccccccccc1111" // claude-code's head
headDu = "dddddddddddd2222" // dunst's head
mergeCC = "553b71910000aaaa" // merged first, 00:20:52
mergeDu = "48bda4750000bbbb" // merged 18 s later, 00:21:09: contains mergeCC
)
var (
atCC = time.Date(2026, 10, 9, 22, 20, 52, 0, time.UTC)
atDu = atCC.Add(17 * time.Second)
)
// twoMerges is the night of issue 362 up to the walks: both pull requests checked, both merged.
func twoMerges(t *testing.T) *world {
w := newWorldOver(t, newMemStore(), newFakeController(), newFakeForge(), atDu.Add(5*time.Second))
cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json")
du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json")
w.h.PullUpdated(cc)
w.h.PullUpdated(du)
w.h.Checked(verdictMoving(cat, 101, headCC, "claude-code"))
w.h.Checked(verdictMoving(cat, 102, headDu, "dunst"))
w.h.PullMerged(mergedAt(cc, mergeCC, atCC))
w.h.PullMerged(mergedAt(du, mergeDu, atDu))
return w
}
// catWalk is a walk of the catalogue at a commit, waiting for its word, moving the modules given.
func catWalk(id, commit string, created time.Time, modules ...string) Walk {
w := Walk{ID: id, Repository: cat, Branch: "main", Commit: commit, Created: created, State: walkBuilding,
Tiers: [][]string{modules}, Modules: map[string]*WalkModule{}, Delivery: &WalkDelivery{Awaits: "mesh-delivery"}}
for _, m := range modules {
w.Modules[m] = &WalkModule{}
}
return w
}
// done is a walk's record once every module passed on its first machine and was sent to the rest.
func done(w Walk, at time.Time) Walk {
w.State = walkDone
for name := range w.Modules {
judged := at
w.Modules[name] = &WalkModule{State: "built", First: []string{"anchor"}, FirstAt: &judged, SentAt: &judged,
Gate: &WalkGate{Machines: []string{"anchor"}, Verdict: "passed", JudgedAt: &judged}}
}
return w
}
func supersededBy(old Walk, by string) Walk {
old.State = walkSuperseded
old.Note = "superseded at tier 0 by " + by + " (novox/mesh-catalog at 48bda475); dunst planned there again"
return old
}
// The replay of issue 362, as today's controller says it: no `merges`, the successor named only in its note. In
// both orders the controller's two records can arrive in, the walk at the dunst commit delivers both merges
// without a person: claude-code covered, dunst delivered, the word asked once.
func TestReplay362(t *testing.T) {
for _, order := range []string{"newer walk first", "supersession first"} {
t.Run(order, func(t *testing.T) {
w := twoMerges(t)
w.forge.contains[mergeCC+" in "+mergeDu] = true
idCC, idDu := IDOf(cat, headCC), IDOf(cat, headDu)
first := w.ctl.put(catWalk("plan-1", mergeDu, atDu.Add(5*time.Second), "dunst"))
w.h.WalkMoved(first)
if s := w.state(idDu); s != Published {
t.Fatalf("dunst with its walk opened is %s", s)
}
second := catWalk("plan-2", mergeDu, atDu.Add(6*time.Second), "claude-code", "dunst")
if order == "newer walk first" {
w.h.WalkMoved(w.ctl.put(second))
w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2")))
} else {
w.ctl.put(second) // kept by the controller, said after
w.h.WalkMoved(w.ctl.put(supersededBy(first, "plan-2")))
}
w.settleAll()
for _, id := range []string{idCC, idDu} {
if d := w.delivery(id); d.State.Final() || d.State == Held {
t.Fatalf("%s is %s (%s)", id, d.State, d.HeldWhy)
}
}
if got := w.ctl.delivers; len(got) != 1 || got[0] != "plan-2" {
t.Fatalf("the word was asked %v, want once for plan-2", got)
}
for _, id := range []string{idCC, idDu} {
if d := w.delivery(id); d.State != Delivering || d.Walk.ID != "plan-2" {
t.Fatalf("%s is %s on %v", id, d.State, d.Walk)
}
}
// Ten minutes and a reconcile later nothing is held for a person.
w.later(11 * time.Minute)
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
w.h.WalkMoved(w.ctl.put(done(w.ctl.walk("plan-2"), w.now)))
w.settleAll()
cc, du := w.delivery(idCC), w.delivery(idDu)
if du.State != Delivered {
t.Fatalf("dunst is %s", du.State)
}
if cc.State != Covered || !strings.Contains(cc.Transitions[len(cc.Transitions)-1].Why, idDu) {
t.Fatalf("claude-code is %s: %+v", cc.State, cc.Transitions[len(cc.Transitions)-1])
}
if len(w.h.Stalled()) != 0 {
t.Fatalf("stalled lists %+v", w.h.Stalled())
}
if len(w.ctl.delivers) != 1 {
t.Fatalf("the word was asked %v", w.ctl.delivers)
}
})
}
}
@@ -83,7 +83,8 @@ const (
EvHistory Event = "history" // made from the forge's word for a merge long past: history
EvRetire Event = "retire" // a person's word that a held delivery is history, never to be walked
EvCovered Event = "covered" // a later merge into the same trunk delivered every module it moves
EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed
EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed, or a walk now answers its merge
EvRewalk Event = "rewalk" // the walk that carried a later commit for it failed: it waits for a walk of its own commit
)
// Facts are what a guard reads beyond the delivery itself: the moment, and who asks, for an act.
@@ -222,8 +223,11 @@ var Table = []Row{
}},
{From: []State{Published}, Event: EvSuperseded, To: Superseded,
Guard: "a newer delivery to the same trunk took over its walk",
Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)",
Holds: func(d *Delivery, f Facts) error {
if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" {
return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy)
}
return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded")
}},
{From: []State{Published}, Event: EvDone, To: Delivered,
@@ -236,8 +240,14 @@ var Table = []Row{
return unless(d.HeldWhy != "" && d.MergedAs != "", "nothing holds it, or it is not on the trunk")
}},
{From: []State{Published}, Event: EvGo, To: Delivering,
Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path",
Holds: func(d *Delivery, f Facts) error { return unless(d.Walk != nil && d.Walk.Started(), "its walk waits") }},
Guard: "its walk started: let go by this owner in its group's order, by a person, or on the controller's own path " +
"— not a failed walk that carried a later commit for it, after which it waits for a walk of its own commit",
Holds: func(d *Delivery, f Facts) error {
if d.Walk != nil && d.Walk.State == walkFailed && d.carriedLater() {
return errors.New("its walk, which carried a later commit for it, failed: it waits for a walk of its own commit")
}
return unless(d.Walk != nil && d.Walk.Started(), "its walk waits")
}},
{From: []State{Held}, Event: EvGo, To: Delivering,
Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " +
"nobody, or a person's `plans go`",
@@ -246,8 +256,8 @@ var Table = []Row{
"its walk was not started by a person or the controller's own path")
}},
{From: []State{Held}, Event: EvLifted, To: Published,
Guard: "held only for its group's composed check, which has since passed or warned for the heads that merged: " +
"it is published again and waits for its turn",
Guard: "held only for its group's composed check, which has since passed or warned for the heads that merged, or " +
"held because no walk answered its merge, and one now does: it is published again and waits for its turn",
Holds: func(d *Delivery, f Facts) error {
return unless(d.Lifted != "" && d.HeldWhy == "" && d.MergedAs != "" && walkCanGoOn(d), "nothing lifted its hold")
},
@@ -259,14 +269,26 @@ var Table = []Row{
}},
{From: []State{Delivering}, Event: EvSuperseded, To: Superseded,
Guard: "a newer delivery to the same trunk took over its walk",
Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)",
Holds: func(d *Delivery, f Facts) error {
if d.Walk != nil && d.Walk.State == walkSuperseded && d.Walk.TakenOverBy != "" {
return fmt.Errorf("its walk was taken over by %s, which answers its merge: it follows that walk", d.Walk.TakenOverBy)
}
return unless(d.Walk != nil && d.Walk.State == walkSuperseded, "its walk is not superseded")
}},
{From: []State{Delivering}, Event: EvDone, To: Delivered,
Guard: "its walk is done: every machine of its deploy plan passed, or was left as its policy says",
Guard: "its walk is done, carrying its own commit: every machine of its deploy plan passed, or was left as its policy says",
Holds: func(d *Delivery, f Facts) error {
return unless(d.Walk != nil && d.Walk.State == walkDone, "its walk is not done")
return unless(d.Walk != nil && d.Walk.State == walkDone && !d.carriedLater(), "its walk is not done, or carried a later commit for it")
}},
{From: []State{Delivering}, Event: EvCovered, To: Covered,
Guard: "its walk is done, carrying a later merge of its branch that contains its merge: answered by its successor in " +
"one walk (novox/hq ADR 0276)",
Holds: func(d *Delivery, f Facts) error {
return unless(d.Walk != nil && d.Walk.State == walkDone && d.carriedLater(), "its walk is not done, or carried its own commit")
},
Says: func(d *Delivery) string {
return "covered: a later merge that contains it was delivered: " + d.Carrier
}},
{From: []State{Delivering}, Event: EvStop, To: Stopped,
Guard: "its walk was stopped through this owner",
@@ -274,9 +296,21 @@ var Table = []Row{
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy != "", "its walk was not stopped")
}},
{From: []State{Delivering}, Event: EvFailed, To: Failed,
Guard: "its walk failed: a gate on a first machine (what it carried put back), a build, a machine",
Guard: "its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine",
Holds: func(d *Delivery, f Facts) error {
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "", "its walk did not fail")
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && !d.carriedLater(),
"its walk did not fail, or carried a later commit for it")
}},
{From: []State{Delivering}, Event: EvRewalk, To: Published,
Guard: "its walk failed carrying a later merge of its branch for it: a failed walk marks no merge delivered, and this " +
"merge waits, published, for a walk of its own commit (novox/hq ADR 0276 decision 3)",
Holds: func(d *Delivery, f Facts) error {
return unless(d.Walk != nil && d.Walk.State == walkFailed && d.Walk.StoppedBy == "" && d.carriedLater(),
"its walk did not fail, or carried its own commit")
},
Says: func(d *Delivery) string {
return fmt.Sprintf("its walk %s, which carried %s for it, failed: it waits for a walk of its own commit",
d.Walk.ID, shortOf(d.AnsweredBy, 12))
}},
{From: notFinal, Event: EvStop, To: Stopped, Act: true,
@@ -450,10 +484,31 @@ func boundSince(d *Delivery) time.Time {
return d.Since
}
// heldUnwalked is the bound of a delivery held because no walk answers its merge (novox/hq issue 362): none. It
// waited the ten minutes a walk opens within already, and nothing but a person moves it now.
var heldUnwalked = Bound{For: 0, Says: "merged, and no walk answers its merge: the operator's"}
// rewalkBound is the bound of a merge whose walk carried a later commit for it and failed: a walk of its own
// commit opens within the ten minutes a walk opens within (ADR 0276 decision 3), or nothing but a person moves it.
var rewalkBound = Bound{For: unwalked, Says: "the walk that carried a later commit for it failed, and no walk of " +
"its own commit opened: the operator's"}
// boundOf is a delivery's bound: its state's, or none for a delivery held because no walk answers it.
func boundOf(d *Delivery) (Bound, bool) {
if d.State == Held && d.Unwalked && d.Walk == nil {
return heldUnwalked, true
}
if d.waitsForItsOwnWalk() {
return rewalkBound, true
}
b, bounded := Bounds[d.State]
return b, bounded
}
// pastBound is whether a delivery has been where it is longer than its state's bound.
func pastBound(d *Delivery, now time.Time) bool {
b, bounded := Bounds[d.State]
return bounded && !d.State.Final() && now.Sub(boundSince(d)) > b.For
b, bounded := boundOf(d)
return bounded && !d.State.Final() && (b.For == 0 || now.Sub(boundSince(d)) > b.For)
}
// StepState is where one machine stands in a delivering walk.
@@ -65,6 +65,14 @@ func aDeliveryFor(r Row) (*Delivery, Facts) {
}
case EvCovered:
d.MergedAs, d.CoveredBy = "aaaa1111", "delivered with novox/app@bbbb (merged as bbbb2222, walk plan-2): app"
if r.From[0] == Delivering {
// Answered by its successor in one walk (ADR 0276): the walk done, carrying a later merge.
d.CoveredBy, d.AnsweredBy, d.Carrier = "", "bbbb2222", "novox/app@bbbb (merged as bbbb2222, walk plan-2)"
d.Walk = &WalkSeen{ID: "plan-2", State: walkDone}
}
case EvRewalk:
d.MergedAs, d.AnsweredBy = "aaaa1111", "bbbb2222"
d.Walk = &WalkSeen{ID: "plan-2", State: walkFailed}
case EvLifted:
d.MergedAs, d.Lifted = "aaaa1111", "its group's composed check passed"
d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"}
@@ -112,7 +120,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing.
func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) {
events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead,
EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop,
EvHistory, EvRetire, EvCovered, EvLifted}
EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk}
for _, from := range append([]State{None}, AllStates...) {
for _, ev := range events {
for _, act := range []bool{false, true} {
@@ -150,8 +158,10 @@ func TestTheTableKeepsItsRules(t *testing.T) {
}
// The trunk rule: only a ready delivery is published, by its merge — or a held one published again when
// what held it is gone (issue 316); nothing off the trunk delivers.
// And a merge whose walk carried a later commit for it and failed waits there for a walk of its own (ADR 0276).
if r.To == Published && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) &&
!(r.Event == EvLifted && len(r.From) == 1 && r.From[0] == Held) {
!(r.Event == EvLifted && len(r.From) == 1 && r.From[0] == Held) &&
!(r.Event == EvRewalk && len(r.From) == 1 && r.From[0] == Delivering) {
t.Errorf("%v —%s→ published: only a ready delivery's merge publishes", r.From, r.Event)
}
// Nothing a person did not decide ends a delivery as stopped or failed by observation from held.
@@ -33,10 +33,23 @@ func (h *Holder) waitsFor(d *Delivery) string {
case Proposed:
return "its check"
case Ready:
if b, batched := h.batchOf(d); batched {
return batchSaid(b, h.Now())
}
if d.MergedAs != "" {
return "a walk that answers its merge"
}
return "its merge"
case Rejected:
return "a new head, or a recheck"
case Published:
if d.waitsForItsOwnWalk() {
return "a walk of its own commit: its walk " + d.Walk.ID + ", which carried " + shortOf(d.AnsweredBy, 8) +
" for it, failed"
}
if d.Walk != nil && d.Walk.TakenOverBy != "" {
return "the walk " + d.Walk.TakenOverBy + ", which took its walk " + d.Walk.ID + " over"
}
if d.Group != "" {
return "its turn in group " + d.Group
}
@@ -154,12 +167,14 @@ func (h *Holder) Groups(all bool) []GroupLine {
type StalledLine struct {
ID string `json:"id"`
// Number is the pull request's, for a head the forge never announced: the condition names it (novox/hq issue 347).
Number int `json:"number,omitempty"`
State State `json:"state"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
Number int `json:"number,omitempty"`
State State `json:"state"`
// Walk is the walk a line of the state Unheld is about (novox/hq issue 362).
Walk string `json:"walk,omitempty"`
For string `json:"for"`
Bound string `json:"bound"`
H2 string `json:"h2"`
Says string `json:"says"`
}
// Stalled is the `stalled` verb: what the controller's self-check raises as `delivery.<id>.stalled`.
@@ -172,7 +187,7 @@ func (h *Holder) Stalled() []StalledLine {
if !pastBound(d, now) {
continue
}
b := Bounds[d.State]
b, _ := boundOf(d)
h2 := "none: the state is the operator's"
switch {
case d.State == Proposed && d.CheckAsk == nil:
@@ -192,6 +207,7 @@ func (h *Holder) Stalled() []StalledLine {
Bound: b.For.String(), H2: h2, Says: b.Says})
}
out = append(out, h.uncheckedLines(now)...)
out = append(out, h.orphanLines(now)...)
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out
}
@@ -355,7 +371,7 @@ func (h *Holder) Close(id, why string) (string, error) {
h.mu.Unlock()
return "", fmt.Errorf("no delivery %q", id)
}
b, bounded := Bounds[d.State]
b, bounded := boundOf(d)
if !bounded || len(b.H2) == 0 {
h.mu.Unlock()
return "", fmt.Errorf("%s is %s: H2 has no transition there — the state is the operator's", id, d.State)