Refuse a send that puts the controller or node-engine behind its machine's build (issue 201)

On 2026-10-02 a plan sent the control node a controller older than the one
running there, whose seat row the newer build had widened; the older binary
refused the row and the mesh went quiet for ten minutes. ADR 0185 took the
outage out of it, and the race stayed open.

Every send but a person's push now refuses a move of mesh-controller or
mesh-host to a build asked before the one its machine was last sent (the
builds' registration order, issue 219), whatever composed it — gated or not.
A gate's own put-back is the one exception, for the modules it puts back,
and says so in its send's scope. The refusal is the send's error, which the
plan keeps as its note, and a warning condition on the machine, cleared by
the next send there that carries no such move.

cmd/mesh-controller/issue201_test.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
+// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
+// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its
+// machine runs, by the builds' registration order.
+
+// askedIn answers when a build was asked from a fixed table: module@commit → when.
+func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) {
+	return func(module, commit string) (time.Time, bool, error) {
+		at, known := table[module+"@"+commit]
+		return at, known, nil
+	}
+}
+
+func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) {
+	old, now := t0, t0.Add(time.Minute)
+	asked := askedIn(map[string]time.Time{
+		"mesh-controller@c1": old, "mesh-controller@c2": now,
+		"mesh-host@h1": old, "mesh-host@h2": now,
+		"app@a1": old, "app@a2": now,
+	})
+	back := []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"},
+		{Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"},
+		{Module: "app", Node: "anchor", From: "a2", To: "a1"},
+	}
+	found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController ||
+		found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine {
+		t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found)
+	}
+	if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") ||
+		!strings.Contains(said, "issue 201") {
+		t.Fatalf("the refusal says the machine, the module and the build: %s", said)
+	}
+
+	// Forward, the same build, or a build the records do not hold: not refused.
+	for _, mv := range []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"},
+		{Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"},
+	} {
+		if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 {
+			t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err)
+		}
+	}
+
+	// A gate's own put-back carries its own modules back, and nothing else.
+	putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
+	found, err = coreDowngrades(back, putBack, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 1 || found[0].move.Module != "mesh-host" {
+		t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
+	}
+	// Naming a module is not a put-back.
+	if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 {
+		t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
+	}
+}
+
+// The refusal is a warning condition on the machine, which the next send there carrying no such move clears.
+func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	was := buildAsked
+	buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) {
+		return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit)
+	}
+	t.Cleanup(func() { buildAsked = was })
+
+	said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}})
+	if err != nil || len(said) != 1 {
+		t.Fatalf("the downgrade is refused: %v %v", said, err)
+	}
+	key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
+	c, open, err := k.Get(ctx, key)
+	if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") {
+		t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err)
+	}
+
+	said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}})
+	if err != nil || len(said) != 0 {
+		t.Fatalf("a move forward is sent: %v %v", said, err)
+	}
+	if _, open, err := k.Get(ctx, key); err != nil || open {
+		t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err)
+	}
+}
+
+// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of
+// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 —
+// is refused, said as the plan's words; the gate's own put-back of the controller is not.
+func TestReplay201(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	record := func(commit string, asked time.Time) inventory.Build {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
+		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
+			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+	before := record("c1aaaaaaaaaa", old)
+	record("c2bbbbbbbbbb", now)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
+		Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
+		t.Fatal(err)
+	}
+	// The mesh's record goes back to the build before, as the record went on 2026-10-02.
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+	wasMoves := machineMoves
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	t.Cleanup(func() { machineMoves = wasMoves })
+
+	judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
+	if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) ||
+		!strings.Contains(err.Error(), "mesh-controller c1aaaaaa") {
+		t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err)
+	}
+	putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})
+	if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
+		t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err)
+	}
+}