Merge pull request 'Keep the controller and node-engine at the build a machine was last sent over an older record (hq issue 201)' (#273) from fix/201-a-push-never-downgrades-the-core into main

cmd/mesh-controller/core_downgrade.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
+// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
+// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
+// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
+// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
+// sent the control node the controller build before it. The older binary read the row it could not serve and the
+// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
+// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
+// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
+//
+// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
+// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
+// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
+// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
+// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
+// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
+// newer merge, or `rebuild mesh-controller`.
+//
+// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
+// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
+// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
+// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
+// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
+// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
+// schema already (issue 352).
+
+// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
+const kindCoreDowngrade = "core-downgrade-kept"
+
+// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
+const coreDowngradeSource = "sends"
+
+// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
+// store and its seat's row, and the node-engine, which reads what the controller declares.
+var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
+
+// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
+type askedBuild struct {
+	asked      time.Time
+	failedGate bool
+}
+
+// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
+// the request time is not known), and whether it failed its gate — and false when the records hold none. A
+// variable, so a test reads its own order.
+var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
+	builds, err := inv.Builds(ctx, module, askedLookBack)
+	if err != nil {
+		return askedBuild{}, false, err
+	}
+	for _, b := range builds {
+		if b.Worked() && sameCommit(b.Commit, commit) {
+			failed, err := inv.GateFailed(ctx, b.ID)
+			if err != nil {
+				return askedBuild{}, false, err
+			}
+			return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
+		}
+	}
+	return askedBuild{}, false, nil
+}
+
+// askedLookBack is how many of a module's newest builds are read for when a build was asked.
+const askedLookBack = 50
+
+// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
+type coreDowngrade struct {
+	module, node, component string
+	// sent is the build the machine was last sent, record the one the mesh holds.
+	sent, record           string
+	sentAsked, recordAsked time.Time
+}
+
+func (d coreDowngrade) String() string {
+	return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
+		"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
+		"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
+		d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
+		d.module)
+}
+
+// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
+// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
+// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
+// but for asked.
+func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
+	asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
+	var out []coreDowngrade
+	for module, was := range sent {
+		component := coreComponent(module)
+		now := record[module]
+		if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
+			continue
+		}
+		if scope.puttingBack && scope.modules[module] {
+			continue
+		}
+		from, known, err := asked(module, was)
+		if err != nil {
+			return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
+		}
+		if !known || from.failedGate {
+			continue
+		}
+		to, known, err := asked(module, now)
+		if err != nil {
+			return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
+		}
+		if known && to.asked.Before(from.asked) {
+			out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
+				sentAsked: from.asked, recordAsked: to.asked})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
+	return out, nil
+}
+
+func guarded(component string) bool {
+	for _, c := range downgradeGuarded {
+		if c == component {
+			return true
+		}
+	}
+	return false
+}
+
+// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
+// was last sent, module → commit, each said and raised as a condition.
+func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
+	current map[string]inventory.CurrentBuild) (map[string]string, error) {
+	record := map[string]string{}
+	for m, c := range current {
+		record[m] = c.Commit
+	}
+	found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
+		return buildAsked(ctx, inv, module, commit)
+	})
+	if err != nil || len(found) == 0 {
+		return nil, err
+	}
+	out := map[string]string{}
+	for _, d := range found {
+		out[d.module] = d.sent
+		sayCoreDowngrade(ctx, d)
+	}
+	return out, nil
+}
+
+// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
+func coreDowngradeObservation(component, node, summary string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
+		Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
+		Summary: summary}
+}
+
+func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
+	fmt.Printf("KEPT: %s\n", d)
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
+		fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
+	}
+}
+
+// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
+// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
+// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
+func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
+	keeper := conditionsFrom
+	if keeper == nil || carried == nil {
+		return
+	}
+	var open []string
+	for _, c := range downgradeGuarded {
+		if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
+			open = append(open, c)
+		}
+	}
+	if len(open) == 0 {
+		return
+	}
+	current, err := inv.CurrentBuilds(ctx)
+	if err != nil {
+		fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
+		return
+	}
+	for module, now := range current {
+		component := coreComponent(module)
+		if !guarded(component) {
+			continue
+		}
+		if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
+			continue
+		}
+		key := coreDowngradeObservation(component, node, "").Key()
+		if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
+			fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
+		}
+	}
+}

cmd/mesh-controller/gate.go

@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 		}
 		return
 	}
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
 	if err != nil {
 		notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
 			"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
 	}
 	inv := open.inventory
 	sort.Strings(b.machines)
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
 	var back []string
 	for _, r := range b.pending {
 		if err != nil {

cmd/mesh-controller/issue201_test.go

@@ -0,0 +1,259 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
+// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
+// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back
+// behind the build its machine was last sent: the send keeps it there, and carries the rest.
+
+// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed
+// their gate.
+func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) {
+	return func(module, commit string) (askedBuild, bool, error) {
+		at, known := table[module+"@"+commit]
+		b := askedBuild{asked: at}
+		for _, f := range failed {
+			if f == module+"@"+commit {
+				b.failedGate = true
+			}
+		}
+		return b, known, nil
+	}
+}
+
+func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) {
+	old, now := t0, t0.Add(time.Minute)
+	table := map[string]time.Time{
+		"mesh-controller@c1": old, "mesh-controller@c2": now,
+		"mesh-host@h1": old, "mesh-host@h2": now,
+		"app@a1": old, "app@a2": now,
+	}
+	sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"}
+	record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"}
+	found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController ||
+		found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine {
+		t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found)
+	}
+	if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") ||
+		!strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") {
+		t.Fatalf("it says the machine, the build kept and the two ways out: %s", said)
+	}
+
+	// The record forward, the same build, or a build the records do not hold: nothing kept.
+	for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} {
+		found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]},
+			map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table))
+		if err != nil || len(found) != 0 {
+			t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err)
+		}
+	}
+
+	// A sent build that failed its gate is never ahead: the record put back before it is sent.
+	if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 ||
+		found[0].module != "mesh-host" {
+		t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found)
+	}
+
+	// A gate's own put-back carries its own modules back, and nothing else.
+	putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
+	if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" {
+		t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
+	}
+	if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}},
+		askedIn(table)); len(found) != 2 {
+		t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
+	}
+}
+
+// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the
+// controller is the build before it, c1, which passed its gate when it rolled out.
+func core201(t *testing.T) *stores {
+	t.Helper()
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	record := func(commit string, asked time.Time) inventory.Build {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
+		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
+			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+	before := record("c1aaaaaaaaaa", old)
+	if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit,
+		Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
+		t.Fatal(err)
+	}
+	record("c2bbbbbbbbbb", now)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
+		Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+	wasMoves := machineMoves
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	t.Cleanup(func() { machineMoves = wasMoves })
+	return open
+}
+
+// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is
+// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer
+// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record,
+// as a person's push does, clears the condition.
+func TestReplay201(t *testing.T) {
+	open := core201(t)
+	ctx := t.Context()
+	k, _ := withConditionsInMemory(t)
+
+	judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
+	if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
+		t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err)
+	}
+	keeps, err := recordedKept(keepingRecorded(judged), open, "anchor")
+	if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" {
+		t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err)
+	}
+	shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}}
+	if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" {
+		t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err)
+	}
+	key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
+	if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") {
+		t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err)
+	}
+
+	putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}))
+	if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 {
+		t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err)
+	}
+
+	settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"})
+	if _, found, _ := k.Get(ctx, key); !found {
+		t.Fatal("a send that kept the newer controller cleared the condition")
+	}
+	settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"})
+	if _, found, _ := k.Get(ctx, key); found {
+		t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open")
+	}
+}
+
+// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was
+// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that
+// failed its gate is never kept.
+func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) {
+	open := core201(t)
+	ctx := t.Context()
+	k, _ := withConditionsInMemory(t)
+	if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb",
+		Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"},
+		Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil {
+		t.Fatal(err)
+	}
+	healer := withScope(ctx, sendScope{})
+	if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
+		t.Fatalf("the healer's send is made: %v %v", kept, err)
+	}
+	if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 {
+		t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err)
+	}
+	if open, _ := k.Open(ctx); len(open) != 0 {
+		t.Fatalf("a completed put-back raised a condition: %+v", open)
+	}
+}
+
+// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept
+// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own
+// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing.
+func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) {
+	open := core201(t)
+	ctx := t.Context()
+	withConditionsInMemory(t)
+	inv := open.inventory
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	for _, b := range []struct {
+		commit string
+		at     time.Time
+	}{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit,
+			Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil {
+			t.Fatal(err)
+		}
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
+			Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit,
+			Asked: b.at}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2",
+		map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil {
+		t.Fatal(err)
+	}
+	// The send as it is made: the controller kept at c2, the app moved to the record.
+	wasSend := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3",
+			map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"})
+	}
+	t.Cleanup(func() { sendRollout = wasSend })
+
+	moves, sent, _, err := gatedSend(ctx, open, "anchor", nil)
+	if err != nil || len(sent) != 1 {
+		t.Fatalf("the gated send is made: %v %v", sent, err)
+	}
+	if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" {
+		t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves)
+	}
+	builds, _, err := inv.SentBuilds(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	g := &inventory.PlanGate{Carried: moves}
+	f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}}
+	for _, m := range []string{"app", "mesh-controller"} {
+		if why, superseded := f.supersededOn(m, "anchor"); superseded {
+			t.Fatalf("the gate is superseded by its own send: %s", why)
+		}
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"register it, so it was not made.", module, m),
 			Resolved: "Resolved: " + module + " on " + m + " is settled"}
 	}),
+	// The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq
+	// issue 201).
+	kindCoreDowngrade: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		component := idPart(o, 0)
+		if o.Scope != conditions.ScopeCore || !guarded(component) {
+			component = "core component"
+		}
+		return words{Headline: m + " keeps a newer " + component + " than the mesh holds",
+			Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m),
+			Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+
+				"the mesh's sends keep the newer one there rather than put it back.", component, m),
+			Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"}
+	}),
 	kindBindingKept: worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		return words{Headline: "A module's data source is held on " + m,

cmd/mesh-controller/push.go

@@ -1027,6 +1027,9 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
 	if err != nil {
 		return "", err
 	}
+	// A send that carried the mesh's record of the controller and the node-engine — a person's push included —
+	// ends the condition of one kept over it (novox/hq issue 201).
+	settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds)
 	if len(s.declared.Bindings) > 0 {
 		// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
 		// what the next resolution keeps it at. On the same outliving context as the send's record.

cmd/mesh-controller/recorded_kept.go

@@ -135,6 +135,17 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
 		}
 		out[m] = was
 	}
+	// **And the controller and the node-engine at the build the machine was last sent, over an older record**
+	// (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already.
+	if !every {
+		core, err := coreKept(ctx, inv, node, sent, current)
+		if err != nil {
+			return nil, err
+		}
+		for m, was := range core {
+			out[m] = was
+		}
+	}
 	return out, nil
 }
 
@@ -159,6 +170,12 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
 			return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
 				"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
 		}
+		if !found && guarded(coreComponent(m)) {
+			return nil, fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+
+				"no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+
+				"a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)",
+				node, m, short(kept[m]), node, m)
+		}
 		if !found {
 			return nil, fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+
 				"records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+

cmd/mesh-controller/release.go

@@ -51,6 +51,9 @@ type sendScope struct {
 	modules map[string]bool
 	// person is a person's act: it carries what it carries.
 	person bool
+	// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
+	// older than the one its machine runs (novox/hq issue 201).
+	puttingBack bool
 }
 
 type sendScopeKey struct{}
@@ -237,6 +240,32 @@ var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node str
 	return f.moves(node, modules, sent, known, all), nil
 }
 
+// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does
+// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the
+// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that
+// call this), so a move it could not keep is still counted.
+func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
+	if len(moves) == 0 {
+		return moves, nil
+	}
+	keeping := ctx
+	if _, on := keptExcept(ctx); !on {
+		keeping = keepingRecorded(ctx)
+	}
+	keeps, err := recordedKept(keeping, open, node)
+	if err != nil || len(keeps) == 0 {
+		return moves, err
+	}
+	var out []inventory.CarriedMove
+	for _, mv := range moves {
+		if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
+			continue
+		}
+		out = append(out, mv)
+	}
+	return out, nil
+}
+
 // ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the
 // machines named, and why; the machine holding the bus, added only for its user list, is left instead.
 func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) {
@@ -254,19 +283,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 		if err != nil {
 			return nil, err
 		}
-		// The grants step composes every module at the build its machine was last sent (novox/hq issue 490):
-		// a move it keeps is not made. Read, not assumed, so a move it could not keep is still refused here.
-		var keeps map[string]string
-		if everyBuildKept(ctx, n) {
-			if keeps, err = recordedKept(ctx, open, n); err != nil {
-				return nil, err
-			}
+		if moves, err = withoutKept(ctx, open, n, moves); err != nil {
+			return nil, err
 		}
 		var waiting []string
 		for _, mv := range moves {
-			if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
-				continue
-			}
 			if !scope.judged[n] && !scope.modules[mv.Module] {
 				waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
 			}
@@ -308,6 +329,11 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
+	// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
+	if moves, err = withoutKept(ctx, open, node, moves); err != nil {
+		return nil, nil, nil, err
+	}
 	own := func(module string) bool {
 		return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
 	}

testdata/store-tests

@@ -31,4 +31,7 @@
 ./cmd/mesh-controller TestADirectoryOwnersKeyIsRefusedThroughAVerb
 ./cmd/mesh-controller TestTheSendMakesNoValueForAnOutsideSecretNobodyGave
 ./cmd/mesh-controller TestAQuestionComposesWithoutAnOutsideSecretNobodyGave
+./cmd/mesh-controller TestReplay201
+./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
+./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
 ./cmd/mesh-controller TestReplay401