Keep a walk's sends to what was kept, and say a silent rest (hq ADR 0353 review)

A release's own gates held its other sends back for ever; an adopted send
lost what the machine ran before; a kept send could grow a build no gate
passed; a take-over matched by commit alone; a rest that went silent ended
done. The node-engine and the resolver still go one machine at a time, since
a fault that cuts a machine off shows only as silence.

cmd/mesh-controller/release_plan.go

@@ -1059,16 +1059,33 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
 	// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
 	// the readings of a gate already judging it there.
-	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 {
+	// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
+	// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
+	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
+		appliedLastSend(ctx, inv, node) {
 		if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
-			adoptSend(ctx, open, p, node, modules, runningOf)
+			adoptSend(ctx, open, p, node, modules, before, runningOf)
 			return nil
 		}
 	}
+	// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
+	// between the send and keeping it reads it here, and a failure still puts the machine back.
+	kept := false
+	for _, m := range modules {
+		s := p.Modules[m]
+		if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
+			s.Previous, kept = before[m], true
+		}
+	}
+	if kept {
+		if err := inv.SavePlan(ctx, p); err != nil {
+			return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
+		}
+	}
 	owns := make([]inventory.CarriedMove, 0, len(modules))
 	for _, m := range modules {
 		s := p.Modules[m]
-		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
+		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
 	}
 	// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
 	// all of it there.
@@ -1083,10 +1100,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	sentWhat := sentNow(ctx, inv, sent)
 	for _, m := range modules {
 		s := p.Modules[m]
-		// What the first machine ran before: what a failed gate puts back (ADR 0236).
-		if beforeKnown {
-			s.Previous = before[m]
-		}
+		// What the first machine ran before (ADR 0236), kept above before the send.
 		s.First, s.FirstAt = sent, &now
 		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
 			From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
@@ -1119,7 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 // from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
 // there — whose carried move also says what the machine ran before, for a rollback.
 func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
-	runningOf map[string][]string) {
+	before map[string]string, runningOf map[string][]string) {
 	inv := open.inventory
 	since := time.Now().UTC()
 	last, found, err := inv.SentTo(ctx, node)
@@ -1127,11 +1141,17 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 		since = last.At
 	}
 	lead := modules[0]
-	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit)
+	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
 	for _, m := range modules {
 		s := p.Modules[m]
-		if m == lead && from != "" {
+		// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
+		// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
+		switch {
+		case s.Previous != "":
+		case m == lead && from != "":
 			s.Previous = from
+		default:
+			s.Previous = before[m]
 		}
 		at := since
 		s.First, s.FirstAt = []string{node}, &at
@@ -1156,13 +1176,34 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	fmt.Printf("%s: %s\n", p.ID, p.Note)
 }
 
+// appliedLastSend says a machine reported it applied the declaration it was sent last.
+func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return false
+	}
+	for _, r := range reports {
+		if r.Node == node {
+			return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
+		}
+	}
+	return false
+}
+
 // judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
 // walk's id, and what the machine ran before; nil when none does.
-func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit string) (*inventory.PlanGate, string, string) {
+func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
 		return nil, "", ""
 	}
+	// The same build, by its id where both name one, else by its commit.
+	same := func(b, c string) bool {
+		if b != "" && build != "" {
+			return b == build
+		}
+		return sameCommit(c, commit)
+	}
 	usable := func(g *inventory.PlanGate) bool {
 		return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
 			slices.Contains(g.Machines, node)
@@ -1181,14 +1222,14 @@ func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, n
 					continue
 				}
 				for _, c := range g.Carried {
-					if c.Module == module && c.Node == node && sameCommit(c.To, commit) {
+					if c.Module == module && c.Node == node && same(c.Build, c.To) {
 						return g, q.ID, c.From
 					}
 				}
 			}
 			continue
 		}
-		if s := q.Modules[module]; s != nil && usable(s.Gate) && sameCommit(s.Commit, commit) {
+		if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
 			return s.Gate, q.ID, s.Previous
 		}
 	}