Send a plan's tier to each machine once, and blame no module for its machine (hq issue 281)

cmd/mesh-controller/gate.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"slices"
+	"sort"
 	"strings"
 	"time"
 
@@ -244,6 +245,86 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string,
 	return healthGood, ""
 }
 
+// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is
+// the judging's facts without those conditions, on what each names among the modules the send moved,
+// and whole what holds the machine back as a whole.
+type machineWord struct {
+	facts gateFacts
+	on    map[string]string
+	whole string
+}
+
+// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not
+// yet reported applying the node tools it was last sent.
+const kindCoreBehind = "core-behind"
+
+// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there
+// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level
+// condition caused by anything else in the send — or by a tier sent one module at a time — failed that
+// module, at the bound, with a reason that was never about it.
+//
+//   - one naming a module the send moved is that module's;
+//   - the core being behind (D10) is the core's: of the node-engine or the node tools when the send
+//     moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about
+//     what was sent: a send not yet applied the machine's reports already say, and a newer core build
+//     that no plan sends is not this send's;
+//   - anything else holds the machine back as a whole: everything the send moved there waits on it, and
+//     fails with it, together, at the bound, with that reason.
+//
+// Pure.
+func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord {
+	w := machineWord{facts: f, on: map[string]string{}}
+	if !f.judged || f.openErr != nil {
+		return w
+	}
+	var core []string
+	for _, m := range moved {
+		if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools {
+			core = append(core, m)
+		}
+	}
+	kept := make([]conditions.Condition, 0, len(f.open))
+	for _, c := range f.open {
+		aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine ||
+			slices.Contains(c.Subject.Also, machine))
+		if !aboutIt || c.Source == gateProbe || c.Raised.Before(since) {
+			kept = append(kept, c)
+			continue
+		}
+		said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary)
+		parts := strings.Split(c.Subject.ID, ".")
+		var named []string
+		for _, m := range moved {
+			if slices.Contains(parts, m) {
+				named = append(named, m)
+			}
+		}
+		coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind)
+		switch {
+		case len(named) > 0:
+			for _, m := range named {
+				if _, already := w.on[m]; !already {
+					w.on[m] = said
+				}
+			}
+		case coreBehind && len(core) > 0:
+			for _, m := range core {
+				if _, already := w.on[m]; !already {
+					w.on[m] = said + " (its settle window runs to the gate's bound)"
+				}
+			}
+		case coreBehind:
+			// Not what the send moved: said nowhere against it.
+		default:
+			if w.whole == "" {
+				w.whole = machine + " as a whole: " + said
+			}
+		}
+	}
+	w.facts.open = kept
+	return w
+}
+
 // judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass
 // counted, a pass missed (and why), or the verdict. Answers the verdict once there is one.
 func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule,
@@ -299,11 +380,30 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
 	if err != nil {
 		return "", err
 	}
+	// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
+	// machine judged, apart from what is wrong with a module there, and never pinned on the module the
+	// gate happens to be kept on.
+	byMachine := map[string][]string{}
+	for _, j := range pairs {
+		byMachine[j.node] = append(byMachine[j.node], j.module)
+	}
+	words := map[string]machineWord{}
+	for node, moved := range byMachine {
+		words[node] = aboutTheMachine(node, moved, *g.Since, facts)
+	}
 	worst, why := healthGood, ""
 	var failing []string
 	broken := map[string]bool{}
 	for _, j := range pairs {
-		h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, facts)
+		w := words[j.node]
+		h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
+		if h == healthGood {
+			if on, named := w.on[j.module]; named {
+				h, said = healthNotYet, on
+			} else if w.whole != "" {
+				h, said = healthNotYet, w.whole
+			}
+		}
 		if h != healthGood && !slices.Contains(failing, j.module) {
 			failing = append(failing, j.module)
 		}
@@ -418,6 +518,21 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 	verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous,
 		Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why,
 		Component: g.Component, JudgingFrom: g.Since}
+	// **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue
+	// 280). The machine already ran this build, or one that made the same artifacts from the same
+	// manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to
+	// put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier
+	// build of the very commit it had failed — "no build kept" — while the build it had run before was
+	// kept all along. Said, left as it is, never marked.
+	if unchangedBy(ctx, inv, module, state.Previous, state.Commit) {
+		g.Rollback = gateUnchanged
+		state.Why = "stopped with its send; the send changed nothing of it: " + g.Why
+		p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+
+			"%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back",
+			strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit))
+		fmt.Printf("%s: %s\n", p.ID, p.Note)
+		return
+	}
 	if state.Build == "" {
 		// A plan from before builds were asked by id: nothing to mark, so nothing is put back by the
 		// mesh — said, for a person.
@@ -450,8 +565,11 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 		sayRollback(ctx, open, module, g, why)
 	}
 	if state.Previous == "" {
-		notBack(fmt.Sprintf("%s had not been sent %s before, or what it was sent is not known — `unassign` takes it "+
-			"off, or a newer merge replaces it", strings.Join(g.Machines, ", "), module))
+		// A first build there: nothing ran before it, so nothing can be put back, and the machine is left
+		// with it — said as that, not as a build the mesh lost.
+		notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+
+			"there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+
+			"a newer merge replaces it", module, strings.Join(g.Machines, ", ")))
 		return
 	}
 	failed, _, err := inv.BuildByID(ctx, state.Build)
@@ -465,7 +583,8 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 		return
 	}
 	if !found {
-		notBack(fmt.Sprintf("no build of %s from %s is still kept to put back", module, short(state.Previous)))
+		notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+
+			"back", module, short(state.Previous), inventory.KeptBuilds))
 		return
 	}
 	if err := inv.RestoreModule(ctx, previous); err != nil {
@@ -476,6 +595,17 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 	if err := inv.SavePlan(ctx, p); err != nil {
 		fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err)
 	}
+	// Put back with the rest of its send, in one send per machine (issue 281).
+	if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched {
+		b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous})
+		b.modules[module] = true
+		for _, n := range g.Machines {
+			if !slices.Contains(b.machines, n) {
+				b.machines = append(b.machines, n)
+			}
+		}
+		return
+	}
 	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
 	if err != nil {
 		notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
@@ -490,6 +620,91 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 	sayRollback(ctx, open, module, g, "")
 }
 
+// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one
+// send judges what it moved as one, and what it found wanting goes back in one send per machine — not
+// in a send for each module, which is the churn that failed the gate in the first place.
+type rollbacks struct {
+	modules  map[string]bool
+	machines []string
+	pending  []pendingRollback
+}
+
+type pendingRollback struct {
+	module   string
+	state    *inventory.PlanModule
+	g        *inventory.PlanGate
+	previous inventory.Build
+}
+
+type rollbacksKey struct{}
+
+// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send
+// to sendRollbacks.
+func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) {
+	b := &rollbacks{modules: map[string]bool{}}
+	return context.WithValue(ctx, rollbacksKey{}, b), b
+}
+
+// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback.
+func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) {
+	if len(b.pending) == 0 {
+		return
+	}
+	inv := open.inventory
+	sort.Strings(b.machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
+	var back []string
+	for _, r := range b.pending {
+		if err != nil {
+			why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
+				"sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines))
+			r.g.Rollback = inventory.NotRolledBack
+			if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil {
+				fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
+			}
+			sayRollback(ctx, open, r.module, r.g, why)
+			continue
+		}
+		r.g.Rollback = inventory.RolledBack
+		back = append(back, r.module+" to "+short(r.previous.Commit))
+		if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil {
+			fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err)
+		}
+		sayRollback(ctx, open, r.module, r.g, "")
+	}
+	if err != nil {
+		p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err)
+		return
+	}
+	p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", "))
+}
+
+// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281):
+// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again.
+const gateUnchanged = "unchanged"
+
+// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the
+// same commit, builds made from the same source, or builds that made the same artifacts from the same
+// manifest. Not known is changed.
+func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool {
+	if from == "" || to == "" {
+		return false
+	}
+	if sameCommit(from, to) {
+		return true
+	}
+	// Made from the same source (issue 280), or the same artifacts from the same manifest.
+	f := moveFacts{}
+	var err error
+	if f.srcs, err = inv.SourceFingerprints(ctx); err != nil {
+		return false
+	}
+	if f.fps, err = inv.Fingerprints(ctx); err != nil {
+		return false
+	}
+	return f.identical(module, from, to)
+}
+
 // rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events.
 type rolledBackEvent struct {
 	Event     string    `json:"event"`