Keep the first-machine gate behind a switch for walks begun before hq ADR 0360, and test both paths

cmd/mesh-controller/release.go

@@ -750,11 +750,49 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 	if r.Rest {
 		return advanceRest(ctx, open, p, now)
 	}
-	if r.Gate == nil {
+	if r.Gate == nil && !gateFirstMachine {
 		// **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step.
 		r.Rest = true
 		return advanceRest(ctx, open, p, now)
 	}
+	if r.Gate == nil {
+		heard, err := releaseHeard(ctx, open)
+		if err != nil {
+			return false, err
+		}
+		for r.Next < len(r.Order) {
+			node := r.Order[r.Next]
+			if !heard[node] {
+				r.Skipped = append(r.Skipped, node)
+				r.Next++
+				continue
+			}
+			moves, err := gatedMoves(ctx, open, node, nil)
+			if errors.Is(err, errWalkedElsewhere) {
+				note := fmt.Sprintf("waiting before %s: %v", node, err)
+				changed := p.Note != note
+				p.Note = note
+				return changed, nil
+			}
+			if err != nil {
+				return false, fmt.Errorf("reading what waits on %s: %w", node, err)
+			}
+			if len(moves) == 0 {
+				r.Done = append(r.Done, node)
+				r.Next++
+				continue
+			}
+			r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
+			p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
+			return true, nil
+		}
+		p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
+		p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
+		if len(r.Skipped) > 0 {
+			p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
+		}
+		return true, nil
+	}
 	g := r.Gate
 	if g.Pending {
 		node := firstOf(g.Machines)
@@ -894,6 +932,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 		}
 	}
 	if len(pending) > 0 {
+		// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
+		// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
+		if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
+			return false, err
+		} else if grew != "" {
+			for _, n := range pending {
+				delete(r.Gates, n)
+			}
+			opened, waiting, err := openRestStep(ctx, open, p)
+			if err != nil {
+				return true, err
+			}
+			p.Note = "chosen again: " + grew
+			if !opened && waiting != "" {
+				p.Note += "; " + waiting
+			}
+			return true, nil
+		}
 		var sent, already, waiting []string
 		for _, node := range pending {
 			out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
@@ -937,6 +993,7 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 	}
 	changed, noteWas := false, p.Note
 	var notes, judging []string
+	batched, back := batchingRollbacks(ctx)
 	for _, node := range r.Order {
 		g := r.Gates[node]
 		if g == nil || g.Verdict != "" || g.Silent {
@@ -963,23 +1020,64 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 			judging = append(judging, node+": not yet reported on its send")
 			continue
 		}
-		// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
-		// fails the walk, with nothing put back — apply-failing says it.
-		switch rep := byNode[node]; rep.Outcome {
-		case inventory.OutcomeApplied:
-			decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
+		if !gateFirstMachine {
+			// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
+			// fails the walk, with nothing put back — apply-failing says it.
+			switch rep := byNode[node]; rep.Outcome {
+			case inventory.OutcomeApplied:
+				decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
+				passCarried(ctx, open, p, g, "")
+				r.Done = append(r.Done, node)
+				changed = true
+			case inventory.OutcomeFailed, inventory.OutcomeRefused:
+				decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
+				r.Failed = append(r.Failed, node)
+				changed = true
+				notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
+			default:
+				judging = append(judging, node+": not yet reported on its send")
+			}
+			continue
+		}
+		verdict, err := judgeMoves(ctx, open, g, nil, now)
+		if err != nil {
+			return changed, err
+		}
+		switch verdict {
+		case "":
+			if putBackBroken(ctx, open, p, g, "", nil) {
+				changed = true
+				notes = append(notes, p.Note)
+			}
+			judging = append(judging, node+": "+gateLine(g))
+		case inventory.GatePassed:
 			passCarried(ctx, open, p, g, "")
 			r.Done = append(r.Done, node)
 			changed = true
-		case inventory.OutcomeFailed, inventory.OutcomeRefused:
-			decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
+		case inventory.GateSuperseded:
+			// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
+			// it; what still waits is released again.
+			r.Done = append(r.Done, node)
+			changed = true
+			notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
+		default:
+			// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
+			// machine of the step goes on to its own verdict.
+			stateWas := p.State
+			p.Note = ""
+			failCarried(batched, open, p, g, "")
+			p.State = stateWas
 			r.Failed = append(r.Failed, node)
 			changed = true
-			notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
-		default:
-			judging = append(judging, node+": not yet reported on its send")
+			notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
+			returnedElsewhere(ctx, open, r, node, g)
 		}
 	}
+	p.Note = ""
+	sendRollbacks(ctx, open, p, back)
+	if p.Note != "" {
+		notes = append(notes, strings.TrimPrefix(p.Note, "; "))
+	}
 	if len(judging) > 0 {
 		note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
 		p.Note = note
@@ -1022,6 +1120,42 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 	return true, nil
 }
 
+// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
+// empty when none would.
+func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
+	if !gateFirstMachine {
+		return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made
+	}
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return "", err
+	}
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return "", err
+	}
+	for _, node := range pending {
+		moves, err := gatedMoves(ctx, open, node, nil)
+		if errors.Is(err, errWalkedElsewhere) {
+			continue // sendKept says it waits
+		}
+		if err != nil {
+			return "", err
+		}
+		kept := r.Gates[node].Carried
+		for _, mv := range moves {
+			// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
+			if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
+				return c.Module == mv.Module && sameCommit(c.To, mv.To)
+			}) {
+				continue
+			}
+			return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
+		}
+	}
+	return "", nil
+}
+
 // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
 // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
 func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
@@ -1034,6 +1168,29 @@ func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bo
 	return r.Current
 }
 
+// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
+// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
+func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
+	for _, c := range g.Carried {
+		if c.Build == "" {
+			continue
+		}
+		if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
+			continue
+		}
+		for node, other := range r.Gates {
+			if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
+				continue
+			}
+			for _, oc := range other.Carried {
+				if oc.Module == c.Module && sameCommit(oc.To, c.To) {
+					other.Returned = append(other.Returned, c.Module)
+				}
+			}
+		}
+	}
+}
+
 // openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
 // every machine left that is heard from and has something waiting, except that a build no gate has passed goes
 // to one machine of the step only. Answers whether a step was kept, or why the walk waits.
@@ -1043,6 +1200,10 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 	if err != nil {
 		return false, "", err
 	}
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return false, "", err
+	}
 	waiting := map[string][]inventory.CarriedMove{}
 	var nodes, walked []string
 	for _, node := range r.Order[min(r.Next, len(r.Order)):] {
@@ -1069,8 +1230,15 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 		waiting[node] = moves
 		nodes = append(nodes, node)
 	}
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return false, "", err
+	}
 	// **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first.
 	step := nodes
+	if gateFirstMachine {
+		step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
+	}
 	if len(step) == 0 {
 		if len(walked) > 0 {
 			return false, "waiting before " + strings.Join(walked, "; "), nil
@@ -1084,9 +1252,50 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 		r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
 	}
 	p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
+	if left := len(nodes) - len(step); left > 0 {
+		p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
+	}
 	return true, "", nil
 }
 
+// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
+// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
+func oneAtATime(module string, m catalogue.Manifest) bool {
+	if module == hostModule {
+		return true
+	}
+	return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
+}
+
+// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
+// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
+// first machine. Pure.
+//
+// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
+// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
+// shows only as its silence and no rollback reaches it.
+func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
+	alone func(module string) bool) []string {
+	first := map[string]string{}
+	var out []string
+	for _, n := range nodes {
+		var unpassed []string
+		for _, mv := range waiting[n] {
+			if !passed(mv.Module, mv.To) || alone(mv.Module) {
+				unpassed = append(unpassed, mv.Module+"@"+mv.To)
+			}
+		}
+		if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
+			continue
+		}
+		for _, k := range unpassed {
+			first[k] = n
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
 // silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
 // §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
 func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {