Walk a stopped walk's merges again in the open batch, in one act, and say what stays on the machines (review of #232, hq issue 459)

A new batch beside the open one walked an older commit after a newer one, and a
stop that failed partway could leave a merge on two plans. A module shared with
the stopped merge, built at its branch, would deliver it; it is left out or
withheld. A batch not yet cut can drop a merge, so mesh-delivery can stop a
merge walked again. The stopped walk names what it left on which machines.

cmd/mesh-controller/delivery.go

@@ -121,22 +121,35 @@ func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string)
 	return p, err
 }
 
-// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the walk stopped, the
-// batch that walks its other merges again, those merges as `<repository>@<commit>`, and every merge left out.
+// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the
+// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk
+// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere
+// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules.
 type stopAnswer struct {
-	Stopped       string                  `json:"stopped"`
-	WalkedAgainBy string                  `json:"walked_again_by,omitempty"`
-	WalkedAgain   []string                `json:"walked_again,omitempty"`
-	LeftOut       []inventory.PlanLeftOut `json:"left_out,omitempty"`
+	Stopped        string                  `json:"stopped"`
+	WalkedAgainBy  string                  `json:"walked_again_by,omitempty"`
+	WalkedAgain    []string                `json:"walked_again,omitempty"`
+	LeftOut        []inventory.PlanLeftOut `json:"left_out,omitempty"`
+	Kept           []inventory.PlanKept    `json:"kept,omitempty"`
+	Withheld       []string                `json:"withheld,omitempty"`
+	StillCarriedBy []string                `json:"still_carried_by,omitempty"`
 }
 
 // stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it
-// answered again in a fresh batch without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its
-// batch, so ending it for one group member ended every other delivery on it. Left out are the merges named and
-// each later merge of their repository's branch, which contains one of them and cannot be walked without
-// delivering it. The fresh batch's window is closed — its merges were heard before — and it is cut by the usual
-// rules, as a failed walk's merges are walked again (ADR 0276 decision 3). With nothing left to walk, or no merge
-// named, the walk is only ended. A merge named that the walk does not answer refuses the stop whole.
+// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending
+// it for one group member ended every other delivery on it.
+//
+//   - Left out are the merges named, each later merge of their repository's branch, which contains one, and each
+//     merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge.
+//   - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older
+//     commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk
+//     withholds the left-out merges' modules where it would build them only as dependents.
+//   - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch
+//     goes on without them; with nothing left in it, the batch itself is stopped.
+//   - The stopped record, the batch and every merge moved are written in one transaction (decision 7).
+//
+// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not
+// answer refuses the stop whole.
 func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string,
 	now time.Time) (inventory.Plan, stopAnswer, error) {
 	said := stopAnswer{Stopped: id}
@@ -149,94 +162,158 @@ func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why
 	if err != nil {
 		return inventory.Plan{}, said, err
 	}
-	if !p.Open() {
+	batch := p.Batch()
+	if !p.Open() && !(batch && len(without) > 0) {
 		return p, said, fmt.Errorf("%s is already %s", p.ID, p.State)
 	}
-	var again []inventory.BatchedMerge
+	var merges, again []inventory.BatchedMerge
 	if len(without) > 0 {
-		merges, err := inv.MergesOf(ctx, p.ID)
-		if err != nil {
+		if merges, err = inv.MergesOf(ctx, p.ID); err != nil {
 			return p, said, err
 		}
 		if again, said.LeftOut, err = leaveOut(merges, without); err != nil {
 			return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
 		}
 	}
-	if p.Delivery == nil {
-		p.Delivery = &inventory.PlanDelivery{}
-	}
-	p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
-	p.Delivery.LeftOut = said.LeftOut
-	p.State = inventory.PlanFailed
-	p.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, p.Tier, why)
-	if len(again) > 0 {
-		batch, err := walkAgain(ctx, inv, p, again, now)
-		if err != nil {
-			return p, said, fmt.Errorf("%s is not stopped: its other merges could not be batched again: %w", p.ID, err)
+	stopped := func(r *inventory.Plan) {
+		if r.Delivery == nil {
+			r.Delivery = &inventory.PlanDelivery{}
 		}
-		said.WalkedAgainBy = batch.ID
-		for _, m := range again {
-			said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
-		}
-		p.Delivery.WalkedAgainBy = batch.ID
-		p.Note += fmt.Sprintf("; its other merges are walked again without %s, by %s: %s", mergesLeftOutWords(said.LeftOut),
-			batch.ID, strings.Join(said.WalkedAgain, ", "))
-		if err := inv.SavePlan(ctx, &p); err != nil {
-			// The merges go back to the walk that still answers them, and the batch holds none: it is never walked.
-			for _, m := range again {
-				_ = inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, m.Alone)
-			}
-			return p, said, err
-		}
-		fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, batch.ID,
-			strings.Join(said.WalkedAgain, ", "))
-		return p, said, nil
+		r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut
+		r.State = inventory.PlanFailed
+		r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why)
 	}
-	if len(said.LeftOut) > 0 {
-		p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
-	}
-	if err := inv.SavePlan(ctx, &p); err != nil {
+	withheld := movesOf(leftOutMerges(merges, said.LeftOut))
+	said.Withheld = withheld
+	if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil {
 		return p, said, err
 	}
+	if batch {
+		return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now)
+	}
+	said.Kept = keptOf(p, movesOf(again))
+	stopped(&p)
+	p.Delivery.Kept = said.Kept
+	if len(again) == 0 {
+		if len(said.LeftOut) > 0 {
+			p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut))
+		}
+		return p, said, inv.SavePlan(ctx, &p)
+	}
+	target, err := batchToWalkAgain(ctx, inv, again, now)
+	if err != nil {
+		return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err)
+	}
+	target.Delivery.Batch.Closed = true
+	target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld)
+	sort.Strings(target.Delivery.Batch.Withheld)
+	said.WalkedAgainBy = target.ID
+	var moves []inventory.MergeMove
+	for _, m := range again {
+		said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
+		moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone})
+	}
+	p.Delivery.WalkedAgainBy = target.ID
+	p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", "))
+	if len(said.LeftOut) > 0 {
+		p.Note += "; left out " + mergesLeftOutWords(said.LeftOut)
+	}
+	if len(said.Kept) > 0 {
+		p.Note += "; " + keptWords(said.Kept)
+	}
+	// The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7).
+	if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&target, &p}, moves); err != nil {
+		return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err)
+	}
+	// What the batch holds and names now: written again by the cutter's next look should this fail.
+	if err := keepBatch(ctx, inv, &target, now, ""); err != nil {
+		fmt.Printf("  %s holds its merges; its record is written again at the next look: %v\n", target.ID, err)
+	}
+	fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID,
+		strings.Join(said.WalkedAgain, ", "))
 	return p, said, nil
 }
 
-// walkAgain puts merges of a stopped walk into a fresh batch of their kind, its window closed: kept before its
-// first merge names it, then each merge moved to it.
-func walkAgain(ctx context.Context, inv *inventory.Inventory, stopped inventory.Plan, again []inventory.BatchedMerge,
+// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which
+// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped
+// itself.
+func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge,
+	said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error {
+	if len(again) == 0 {
+		stopped(b)
+		b.Delivery.Batch = nil
+		b.Note = "a batch whose every merge was left out by a stop: " + b.Note
+		return inv.SavePlan(ctx, b)
+	}
+	left := leftOutMerges(merges, said.LeftOut)
+	first := left[0]
+	r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
+		Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{},
+		Modules: map[string]*inventory.PlanModule{}}
+	if r.ID == b.ID {
+		r.ID += "-left"
+	}
+	stopped(&r)
+	r.Delivery.WalkedAgainBy = b.ID
+	r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID,
+		mergesLeftOutWords(said.LeftOut), b.ID)
+	b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld)
+	sort.Strings(b.Delivery.Batch.Withheld)
+	var moves []inventory.MergeMove
+	for _, m := range left {
+		moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone})
+	}
+	for _, m := range again {
+		said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit)
+	}
+	said.Stopped, said.WalkedAgainBy = r.ID, b.ID
+	if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil {
+		return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err)
+	}
+	if err := keepBatch(ctx, inv, b, now, ""); err != nil {
+		fmt.Printf("  %s holds its merges; its record is written again at the next look: %v\n", b.ID, err)
+	}
+	fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID)
+	return nil
+}
+
+// batchToWalkAgain is the batch a stopped walk's merges join: the open batch of their kind, or a new one, not yet
+// written. The kind is read from what the merges moved.
+func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, again []inventory.BatchedMerge,
 	now time.Time) (inventory.Plan, error) {
 	own := false
-	for _, m := range again {
-		for _, n := range announcedOf(m).Moves {
-			if _, on := onTheControllersPath[n]; on {
-				own = true
+	for _, n := range movesOf(again) {
+		if _, on := onTheControllersPath[n]; on {
+			own = true
+		}
+	}
+	batches, err := inv.Batches(ctx)
+	if err != nil {
+		return inventory.Plan{}, err
+	}
+	for _, b := range batches {
+		if b.OwnPath() == own {
+			if b.Delivery == nil {
+				b.Delivery = &inventory.PlanDelivery{}
 			}
+			if b.Delivery.Batch == nil {
+				b.Delivery.Batch = &inventory.PlanBatch{Own: own}
+			}
+			return b, nil
 		}
 	}
 	first := again[0]
-	b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
+	return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch,
 		Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling,
 		Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{},
-		Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}
-	if b.ID == stopped.ID {
-		b.ID += "-again"
-	}
-	if err := inv.SavePlan(ctx, &b); err != nil {
-		return b, err
-	}
-	for _, m := range again {
-		if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, b.ID, m.Alone); err != nil {
-			return b, err
-		}
-	}
-	// Its merges were heard before: the window, measured from the last of them, is closed, and the cutter cuts it
-	// once no walk of its kind is open.
-	return b, keepBatch(ctx, inv, &b, now, "")
+		Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil
 }
 
 // leaveOut splits a walk's merges by the merges named to leave out, each `<repository>@<commit>` (a commit's
-// first characters will do): the merges walked again, and those left out — each named one, and each later merge
-// of its repository's branch, which contains it. A name the walk does not answer is refused.
+// first characters will do): the merges walked again, and those left out — each named one; each later merge of
+// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is
+// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is
+// refused.
 func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) {
 	var stopped []inventory.BatchedMerge
 	for _, w := range without {
@@ -252,40 +329,181 @@ func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.Ba
 		}
 		stopped = append(stopped, merges[i])
 	}
-	// The earliest stopped merge of each repository's branch: every merge of that branch after it contains it.
-	earliest := map[string]inventory.BatchedMerge{}
-	for _, s := range stopped {
-		k := strings.ToLower(s.Repository) + "\x00" + s.Branch
-		if e, ok := earliest[k]; !ok || laterMerge(e, s) {
-			earliest[k] = s
+	same := func(a, b inventory.BatchedMerge) bool {
+		return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit
+	}
+	out := map[int]inventory.PlanLeftOut{}
+	for i, m := range merges {
+		if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) {
+			out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit}
+		}
+	}
+	for grew := true; grew; {
+		grew = false
+		for i, m := range merges {
+			if _, left := out[i]; left {
+				continue
+			}
+			for j, o := range out {
+				l := merges[j]
+				contains := o.Contains
+				if contains == "" {
+					contains = l.Commit
+				}
+				if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) {
+					out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains}
+				} else if shared := sharedMoves(m, l); len(shared) > 0 {
+					out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains,
+						Shares: shared}
+				} else {
+					continue
+				}
+				grew = true
+				break
+			}
 		}
 	}
 	var again []inventory.BatchedMerge
-	var out []inventory.PlanLeftOut
-	for _, m := range merges {
-		named := slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool {
-			return strings.EqualFold(s.Repository, m.Repository) && s.Commit == m.Commit
-		})
-		e, has := earliest[strings.ToLower(m.Repository)+"\x00"+m.Branch]
-		switch {
-		case named:
-			out = append(out, inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit})
-		case has && laterMerge(m, e):
-			out = append(out, inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: e.Commit})
-		default:
+	var left []inventory.PlanLeftOut
+	for i, m := range merges {
+		if o, is := out[i]; is {
+			left = append(left, o)
+		} else {
 			again = append(again, m)
 		}
 	}
-	sort.SliceStable(out, func(i, j int) bool { return out[i].Contains == "" && out[j].Contains != "" })
-	return again, out, nil
+	sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" })
+	return again, left, nil
 }
 
-// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one says so.
+// sharedMoves are the modules two kept merges both moved when they were heard.
+func sharedMoves(a, b inventory.BatchedMerge) []string {
+	var out []string
+	theirs := announcedOf(b).Moves
+	for _, n := range announcedOf(a).Moves {
+		if slices.Contains(theirs, n) && !slices.Contains(out, n) {
+			out = append(out, n)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// leftOutMerges are the kept merges a stop left out.
+func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge {
+	var left []inventory.BatchedMerge
+	for _, m := range merges {
+		if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool {
+			return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit
+		}) {
+			left = append(left, m)
+		}
+	}
+	return left
+}
+
+// movesOf is every module kept merges moved when they were heard, sorted.
+func movesOf(merges []inventory.BatchedMerge) []string {
+	var out []string
+	for _, m := range merges {
+		out = union(out, announcedOf(m).Moves)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached:
+// those keep running the stopped walk's build.
+func keptOf(p inventory.Plan, walkedAgain []string) []inventory.PlanKept {
+	var out []inventory.PlanKept
+	for name, m := range p.Modules {
+		if m == nil || slices.Contains(walkedAgain, name) {
+			continue
+		}
+		var machines []string
+		if m.FirstAt != nil || m.SentAt != nil {
+			machines = union(machines, m.First)
+		}
+		if m.Gate != nil && m.Gate.Since != nil {
+			machines = union(machines, m.Gate.Machines)
+		}
+		for machine := range m.Rest {
+			machines = union(machines, []string{machine})
+		}
+		if len(machines) == 0 {
+			continue
+		}
+		sort.Strings(machines)
+		out = append(out, inventory.PlanKept{Module: name, Machines: machines})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
+	return out
+}
+
+// keptWords is what a stopped walk leaves on the machines, as a person reads it.
+func keptWords(kept []inventory.PlanKept) string {
+	var words []string
+	for _, k := range kept {
+		words = append(words, k.Module+" on "+strings.Join(k.Machines, ", "))
+	}
+	return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ")
+}
+
+// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped
+// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a
+// module it moves. Holding a merged change back for good is a revert's (ADR 0299).
+func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge,
+	out []inventory.PlanLeftOut, withheld []string) ([]string, error) {
+	var named []inventory.BatchedMerge
+	for _, m := range leftOutMerges(merges, out) {
+		if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) {
+			named = append(named, m)
+		}
+	}
+	if len(named) == 0 {
+		return nil, nil
+	}
+	open, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, err
+	}
+	batches, err := inv.Batches(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var carried []string
+	for _, p := range append(open, batches...) {
+		if p.ID == id {
+			continue
+		}
+		ms, err := inv.MergesOf(ctx, p.ID)
+		if err != nil {
+			return nil, err
+		}
+		for _, m := range ms {
+			later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool {
+				return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s)
+			})
+			shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) })
+			if later || shares {
+				carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID))
+			}
+		}
+	}
+	sort.Strings(carried)
+	return carried, nil
+}
+
+// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or
+// shares its modules, says so.
 func mergesLeftOutWords(out []inventory.PlanLeftOut) string {
 	var words []string
 	for _, o := range out {
 		s := o.Repository + "@" + short(o.Commit)
-		if o.Contains != "" {
+		switch {
+		case len(o.Shares) > 0:
+			s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)"
+		case o.Contains != "":
 			s += " (which contains " + short(o.Contains) + ")"
 		}
 		words = append(words, s)