Send a plan's tier to each machine once, and blame no module for its machine (hq issue 281)

cmd/mesh-controller/release_plan.go

@@ -610,7 +610,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// minute. Now the first machine is sent, the plan records it and waits for that machine's report
 	// after the send to say it applied what it was sent; only then are the rest sent. A first machine
 	// that fails or refuses stops the module's rollout and the plan with it, the rest untouched.
+	//
+	// **One send per machine for the whole tier** (novox/hq issue 281). A send carries the machine's
+	// whole declaration (ADR 0221), and the plan sent the first machine once per module: on 2026-10-06 a
+	// tier of seventy modules sent one laptop a new declaration every few seconds, the node-engine set
+	// aside one for the next, the node tools never settled, and the gate blamed a module for the machine
+	// being behind. Now every module of the tier whose first machine is the same is sent there in one
+	// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
+	// of the tier's machines are sent once, together, when the gates have passed.
 	var pending []string
+	firstOn := map[string][]string{}
+	runningOf := map[string][]string{}
+	var rest []string
+	restTo := map[string][]string{}
+	together := map[string]bool{}
 	for _, m := range tier {
 		state := p.Modules[m]
 		if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) {
@@ -620,6 +633,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 		if err != nil {
 			return false, err
 		}
+		runningOf[m] = running
 		// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
 		// of the build every machine running it was sent, it is registered with that build's artifacts —
 		// nothing to send, and nothing for a gate to judge.
@@ -647,25 +661,37 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 		}
 		now := time.Now().UTC()
 		step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound)
+		// **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its
+		// verdict on its first machine. A failure there stopped the plan already.
+		if state.GatedBy != "" {
+			lead := p.Modules[state.GatedBy]
+			if lead == nil || lead.Gate == nil || lead.Gate.Verdict == "" {
+				pending = append(pending, fmt.Sprintf("%s judged with %s on %s", m, state.GatedBy, strings.Join(state.First, ", ")))
+				continue
+			}
+			if lead.Gate.Verdict != inventory.GatePassed {
+				continue
+			}
+			passedWith(state, state.GatedBy, lead.Gate)
+		}
 		switch {
 		case step.failed != "":
 			// The first machine refused or failed what it was sent, or never said: the gate failed, and
-			// the build is put back there (novox/hq ADR 0236); the rest are left as they were.
-			gateFailed(ctx, open, p, m, state, firstRunning(state.First, running), step.failed)
-			if state.Gate != nil && len(state.Gate.Carried) > 0 {
-				failCarried(ctx, open, p, state.Gate, m)
-			}
-			p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
-			fmt.Printf("%s: %s\n", p.ID, p.Note)
+			// what it carried is put back there (novox/hq ADR 0236); the rest are left as they were.
+			failFirstSend(ctx, open, p, m, state, firstRunning(state.First, running), step.failed, step.rest)
 			return true, nil
 		case step.waiting != "":
-			pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, state.FirstAt.Local().Format("15:04")))
+			at := ""
+			if state.FirstAt != nil {
+				at = state.FirstAt.Local().Format("15:04")
+			}
+			pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, at))
 			continue
 		}
 		// **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied;
 		// it is judged by its health before anything else is sent — the rest, or, where it is the only
 		// machine, the plan's next step.
-		if !policy.Together && state.FirstAt != nil && len(state.First) > 0 {
+		if state.GatedBy == "" && !policy.Together && state.FirstAt != nil && len(state.First) > 0 {
 			verdict, err := judgeGate(ctx, open, p, m, state, running, now)
 			if err != nil {
 				return false, err
@@ -676,12 +702,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 					strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
 				continue
 			case inventory.GateFailed:
-				gateFailed(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why)
-				if len(state.Gate.Carried) > 0 {
-					failCarried(ctx, open, p, state.Gate, m)
-				}
-				p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(step.rest, ", ")))
-				fmt.Printf("%s: %s\n", p.ID, p.Note)
+				failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest)
 				return true, nil
 			case inventory.GatePassed:
 				if !state.Gate.Kept {
@@ -695,51 +716,66 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			// No machine runs it: nothing to send, and nothing to wait for.
 			state.SentAt = &now
 			continue
-		}
-		// Read before the send, which records what it carries.
-		var before map[string]string
-		var beforeKnown bool
-		if step.first {
-			before, beforeKnown, _ = inv.SentBuilds(ctx, step.send[0])
-		}
-		// What sendToEach answers, not what was asked: the machine holding the bus is sent before
-		// the first when its user list must change (issue 249), and the plan waits for it too.
-		var sent []string
-		var carried []inventory.CarriedMove
-		switch {
 		case step.first:
-			// **A gated send** (ADR 0236): everything waiting on the first machine goes with the build,
-			// and the gate judges all of it there.
-			own := inventory.CarriedMove{Module: m, Node: step.send[0], From: before[m], To: state.Commit, Build: state.Build}
-			carried, sent, err = gatedSend(ctx, open, step.send[0], &own)
-		case policy.Together:
-			sent, err = sendRollout(withScope(ctx, sendScope{modules: map[string]bool{m: true}}), open, step.send)
-		default:
-			// The rest, after the gate passed: nothing else may move with it that no gate has seen.
-			sent, err = sendRollout(ctx, open, step.send)
+			firstOn[step.send[0]] = append(firstOn[step.send[0]], m)
+			continue
 		}
-		if err != nil {
-			// Not marked sent, so the next step tries again (issue 249): a grant that could not be
-			// issued is a send that did not happen.
-			return false, fmt.Errorf("sending %s to %s after tier %d: %w", m, strings.Join(step.send, ", "), p.Tier, err)
+		rest = append(rest, m)
+		restTo[m] = step.send
+		together[m] = policy.Together
+	}
+	// The first machines: one send each for everything of the tier that goes there first, one machine
+	// per step — the plan is kept between them, so the next machine's send sees what this one walks.
+	if len(firstOn) > 0 {
+		machines := make([]string, 0, len(firstOn))
+		for n := range firstOn {
+			machines = append(machines, n)
 		}
-		if step.first {
-			// What the first machine ran before: what a failed gate puts back (ADR 0236).
-			if beforeKnown {
-				state.Previous = before[m]
+		sort.Strings(machines)
+		for _, node := range machines {
+			err := firstSend(ctx, open, p, node, firstOn[node], runningOf)
+			if errors.Is(err, errWalkedElsewhere) {
+				// A build this machine would carry is being judged elsewhere: it goes once that gate passed.
+				pending = append(pending, fmt.Sprintf("%s for %s: %v", node, strings.Join(firstOn[node], ", "), err))
+				continue
+			}
+			if err != nil {
+				// Not marked sent, so the next step tries again (issue 249): a grant that could not be
+				// issued is a send that did not happen.
+				return false, fmt.Errorf("sending %s to %s first after tier %d: %w", strings.Join(firstOn[node], ", "),
+					node, p.Tier, err)
 			}
-			state.First = sent
-			state.FirstAt = &now
-			state.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, running),
-				From: state.Previous, To: state.Commit, Since: &now, Carried: carried}
-			p.State = inventory.PlanRolling
-			p.Note = fmt.Sprintf("tier %d built; sent %s to %s first", p.Tier, m, strings.Join(sent, ", "))
-			fmt.Printf("%s: tier %d built; sent %s to %s first, the rest once it reports it applied\n",
-				p.ID, p.Tier, m, strings.Join(sent, ", "))
 			return true, nil
 		}
-		state.SentAt = &now
-		fmt.Printf("%s: tier %d built; sent %s to %s\n", p.ID, p.Tier, m, strings.Join(sent, ", "))
+	}
+	// The rest, after their gates passed, and what rolls out together: every machine once.
+	if len(rest) > 0 {
+		var machines []string
+		scope := sendScope{modules: map[string]bool{}}
+		for _, m := range rest {
+			for _, n := range restTo[m] {
+				if !slices.Contains(machines, n) {
+					machines = append(machines, n)
+				}
+			}
+			if together[m] {
+				// What its policy rolls out together may move wherever it goes; anything else no gate has
+				// seen still stops the send.
+				scope.modules[m] = true
+			}
+		}
+		sort.Strings(machines)
+		sent, err := sendRollout(withScope(ctx, scope), open, machines)
+		if err != nil {
+			return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "),
+				strings.Join(machines, ", "), p.Tier, err)
+		}
+		now := time.Now().UTC()
+		for _, m := range rest {
+			p.Modules[m].SentAt = &now
+		}
+		fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
+			strings.Join(sent, ", "))
 		return true, nil
 	}
 	if len(pending) > 0 {
@@ -817,6 +853,91 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	return true, nil
 }
 
+// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is
+// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once,
+// before the send, so a module the same send carries is never read as already moved — the machines it
+// went to, and the gate. The gate is kept on the first of them and judges everything the send moved;
+// the others are gated by it.
+func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
+	runningOf map[string][]string) error {
+	inv := open.inventory
+	before, beforeKnown, _ := inv.SentBuilds(ctx, node)
+	owns := make([]inventory.CarriedMove, 0, len(modules))
+	for _, m := range modules {
+		s := p.Modules[m]
+		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
+	}
+	// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
+	// all of it there.
+	carried, sent, err := gatedSend(ctx, open, node, owns)
+	if err != nil {
+		return err
+	}
+	now := time.Now().UTC()
+	lead := modules[0]
+	for _, m := range modules {
+		s := p.Modules[m]
+		// What the first machine ran before: what a failed gate puts back (ADR 0236).
+		if beforeKnown {
+			s.Previous = before[m]
+		}
+		s.First, s.FirstAt = sent, &now
+		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
+			From: s.Previous, To: s.Commit, Since: &now}
+		s.GatedBy = ""
+		if m == lead {
+			s.Gate.Carried = carried
+		} else {
+			s.GatedBy = lead
+		}
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
+		strings.Join(sent, ", "))
+	fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n",
+		p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", "))
+	return nil
+}
+
+// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
+// on the first of them passed, and its pass was kept for every build it carried (passCarried).
+func passedWith(s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
+	if s.Gate == nil {
+		s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since}
+	}
+	if s.Gate.Verdict != "" {
+		return
+	}
+	s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+g.Why, g.JudgedAt, g.Took
+	s.Gate.Passes, s.Gate.Kept = g.Passes, true
+}
+
+// failFirstSend stops the plan at a first send whose gate failed, or whose machine refused or failed
+// what it was sent: what the gate found wanting is put back there, each once — the module the gate is
+// kept on only when it is among them (issue 281: the module whose gate it is was blamed for whatever
+// the send carried) — and the machines after it are left as they were.
+func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
+	machines []string, why string, rest []string) {
+	batched, back := batchingRollbacks(ctx)
+	defer func() {
+		sendRollbacks(ctx, open, p, back)
+		p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(rest, ", ")))
+		fmt.Printf("%s: %s\n", p.ID, p.Note)
+	}()
+	g := state.Gate
+	if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
+		gateFailed(batched, open, p, module, state, machines, why)
+	} else {
+		state.Why = "not found wanting; stopped with the send that carried it: " + g.Why
+		p.State = inventory.PlanFailed
+		p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was not found wanting and is left as it is",
+			strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
+	}
+	if state.Gate != nil && len(state.Gate.Carried) > 0 {
+		failCarried(batched, open, p, state.Gate, module)
+	}
+}
+
 // rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249).
 type rolloutStep struct {
 	// send is the machines to send now; first, whether they are the first machine's send.
@@ -1139,7 +1260,11 @@ func plansCommand(ctx context.Context, args []string) error {
 				fmt.Printf("    %-22s %s\n", m, state)
 				// The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to,
 				// verdict, time to it, rolled back or not.
-				if s != nil && s.Gate != nil {
+				switch {
+				case s != nil && s.GatedBy != "" && (s.Gate == nil || s.Gate.Verdict == ""):
+					// Sent with others, judged by the gate of that send (issue 281).
+					fmt.Printf("    %-22s judged with %s, in the send to %s\n", "", s.GatedBy, strings.Join(s.First, ", "))
+				case s != nil && s.Gate != nil:
 					fmt.Printf("    %-22s %s\n", "", gateLine(s.Gate))
 				}
 			}