Refuse a send that puts the controller or node-engine behind its machine's build (issue 201)

On 2026-10-02 a plan sent the control node a controller older than the one
running there, whose seat row the newer build had widened; the older binary
refused the row and the mesh went quiet for ten minutes. ADR 0185 took the
outage out of it, and the race stayed open.

Every send but a person's push now refuses a move of mesh-controller or
mesh-host to a build asked before the one its machine was last sent (the
builds' registration order, issue 219), whatever composed it — gated or not.
A gate's own put-back is the one exception, for the modules it puts back,
and says so in its send's scope. The refusal is the send's error, which the
plan keeps as its note, and a warning condition on the machine, cleared by
the next send there that carries no such move.

cmd/mesh-controller/core_downgrade.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// **A send never moves the mesh's own core to a build older than the one its machine runs** (novox/hq issue
+// 201). On 2026-10-02 two merges a minute apart made two plans; the controller's plan rolled the new controller,
+// which widened its seat's row, and the node-engine's plan then sent the control node the controller build
+// before it. The older binary read the row it could not serve and the mesh lost its voice for ten minutes — the
+// thing that would have corrected it was the thing that was down. ADR 0185 took the outage out of it (a
+// controller behind its row serves what it can); the race is closed here, without needing its trigger: whatever
+// composed the older build, the send that would carry it is refused.
+//
+// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the
+// commits' dates. A gate's own put-back is the one send that may carry it, for the modules it puts back: it says
+// so (sendScope.puttingBack), and the controller's put-back is held to the store's schema already (issue 352).
+// A person's push carries what it carries, as every other refusal of a send leaves it.
+//
+// The refusal is said in the plan's words — the send's error, which the plan keeps as its note and tries again
+// — and as a warning condition on the machine, cleared by the first send there that carries no such move.
+
+// errCoreDowngrade is a send refused because it would put a core module on a machine back behind the build that
+// machine runs.
+var errCoreDowngrade = errors.New("a send would put the mesh's own core back behind the build its machine runs")
+
+// kindCoreDowngrade is that refusal, as a condition.
+const kindCoreDowngrade = "core-downgrade-refused"
+
+// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
+const coreDowngradeSource = "sends"
+
+// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
+// store and its seat's row, and the node-engine, which reads what the controller declares.
+var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
+
+// buildAsked is when the newest build of a module from a commit that worked was asked — or recorded, when the
+// request time is not known — and false when the records hold none. A variable, so a test reads its own order.
+var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (time.Time, bool, error) {
+	builds, err := inv.Builds(ctx, module, askedLookBack)
+	if err != nil {
+		return time.Time{}, false, err
+	}
+	for _, b := range builds {
+		if b.Worked() && sameCommit(b.Commit, commit) {
+			return b.AskedOrAt(), true, nil
+		}
+	}
+	return time.Time{}, false, nil
+}
+
+// askedLookBack is how many of a module's newest builds are read for when a build was asked.
+const askedLookBack = 50
+
+// coreDowngrade is one move of a core module to an older build than its machine runs.
+type coreDowngrade struct {
+	move             inventory.CarriedMove
+	component        string
+	fromAsked, asked time.Time
+}
+
+func (d coreDowngrade) String() string {
+	return fmt.Sprintf("%s would be sent %s %s, older than the %s it runs (asked %s, before %s) — no send but a "+
+		"gate's own put-back moves the %s back (novox/hq issue 201)", d.move.Node, d.move.Module, short(d.move.To),
+		short(d.move.From), d.asked.UTC().Format(time.RFC3339), d.fromAsked.UTC().Format(time.RFC3339), d.component)
+}
+
+// coreDowngrades is the moves of a machine's send that put a core module back to a build asked before the one it
+// runs, leaving out what a put-back puts back. A move whose either build the records do not hold is not proved
+// older, and is left to the gate that judges it. Pure but for asked.
+func coreDowngrades(moves []inventory.CarriedMove, scope sendScope,
+	asked func(module, commit string) (time.Time, bool, error)) ([]coreDowngrade, error) {
+	var out []coreDowngrade
+	for _, mv := range moves {
+		component := coreComponent(mv.Module)
+		if !guarded(component) || mv.From == "" || mv.To == "" || sameCommit(mv.From, mv.To) {
+			continue
+		}
+		if scope.puttingBack && scope.modules[mv.Module] {
+			continue
+		}
+		from, known, err := asked(mv.Module, mv.From)
+		if err != nil || !known {
+			if err != nil {
+				return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.From), err)
+			}
+			continue
+		}
+		to, known, err := asked(mv.Module, mv.To)
+		if err != nil {
+			return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.To), err)
+		}
+		if known && to.Before(from) {
+			out = append(out, coreDowngrade{move: mv, component: component, fromAsked: from, asked: to})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].move.Module < out[j].move.Module })
+	return out, nil
+}
+
+func guarded(component string) bool {
+	for _, c := range downgradeGuarded {
+		if c == component {
+			return true
+		}
+	}
+	return false
+}
+
+// refuseCoreDowngrades says, for one machine's moves, each that would put a core module back, raising its
+// condition; and clears the condition of each guarded component that machine's send does not move back.
+func refuseCoreDowngrades(ctx context.Context, inv *inventory.Inventory, scope sendScope, node string,
+	moves []inventory.CarriedMove) ([]string, error) {
+	found, err := coreDowngrades(moves, scope, func(module, commit string) (time.Time, bool, error) {
+		return buildAsked(ctx, inv, module, commit)
+	})
+	if err != nil {
+		return nil, err
+	}
+	var said []string
+	refused := map[string]bool{}
+	for _, d := range found {
+		said = append(said, d.String())
+		refused[d.component] = true
+		sayCoreDowngrade(ctx, d)
+	}
+	for _, c := range downgradeGuarded {
+		if !refused[c] {
+			clearCoreDowngrade(ctx, c, node)
+		}
+	}
+	return said, nil
+}
+
+// coreDowngradeObservation is a refused downgrade as a condition: `core.<component>.<machine>.<kind>`.
+func coreDowngradeObservation(component, node, summary string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
+		Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
+		Summary: summary}
+}
+
+func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
+	fmt.Printf("NOT SENT: %s\n", d)
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.move.Node, d.String())); err != nil {
+		fmt.Printf("the condition for %s's refused downgrade on %s could not be raised: %v\n", d.move.Module,
+			d.move.Node, err)
+	}
+}
+
+func clearCoreDowngrade(ctx context.Context, component, node string) {
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	key := coreDowngradeObservation(component, node, "").Key()
+	if _, err := keeper.Clear(ctx, key, "a send to "+node+" carries no older "+component); err != nil {
+		fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
+	}
+}

cmd/mesh-controller/gate.go

@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 		}
 		return
 	}
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
 	if err != nil {
 		notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
 			"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
 	}
 	inv := open.inventory
 	sort.Strings(b.machines)
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
 	var back []string
 	for _, r := range b.pending {
 		if err != nil {

cmd/mesh-controller/issue201_test.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
+// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
+// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its
+// machine runs, by the builds' registration order.
+
+// askedIn answers when a build was asked from a fixed table: module@commit → when.
+func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) {
+	return func(module, commit string) (time.Time, bool, error) {
+		at, known := table[module+"@"+commit]
+		return at, known, nil
+	}
+}
+
+func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) {
+	old, now := t0, t0.Add(time.Minute)
+	asked := askedIn(map[string]time.Time{
+		"mesh-controller@c1": old, "mesh-controller@c2": now,
+		"mesh-host@h1": old, "mesh-host@h2": now,
+		"app@a1": old, "app@a2": now,
+	})
+	back := []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"},
+		{Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"},
+		{Module: "app", Node: "anchor", From: "a2", To: "a1"},
+	}
+	found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController ||
+		found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine {
+		t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found)
+	}
+	if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") ||
+		!strings.Contains(said, "issue 201") {
+		t.Fatalf("the refusal says the machine, the module and the build: %s", said)
+	}
+
+	// Forward, the same build, or a build the records do not hold: not refused.
+	for _, mv := range []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"},
+		{Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"},
+	} {
+		if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 {
+			t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err)
+		}
+	}
+
+	// A gate's own put-back carries its own modules back, and nothing else.
+	putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
+	found, err = coreDowngrades(back, putBack, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 1 || found[0].move.Module != "mesh-host" {
+		t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
+	}
+	// Naming a module is not a put-back.
+	if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 {
+		t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
+	}
+}
+
+// The refusal is a warning condition on the machine, which the next send there carrying no such move clears.
+func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	was := buildAsked
+	buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) {
+		return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit)
+	}
+	t.Cleanup(func() { buildAsked = was })
+
+	said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}})
+	if err != nil || len(said) != 1 {
+		t.Fatalf("the downgrade is refused: %v %v", said, err)
+	}
+	key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
+	c, open, err := k.Get(ctx, key)
+	if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") {
+		t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err)
+	}
+
+	said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}})
+	if err != nil || len(said) != 0 {
+		t.Fatalf("a move forward is sent: %v %v", said, err)
+	}
+	if _, open, err := k.Get(ctx, key); err != nil || open {
+		t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err)
+	}
+}
+
+// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of
+// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 —
+// is refused, said as the plan's words; the gate's own put-back of the controller is not.
+func TestReplay201(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	record := func(commit string, asked time.Time) inventory.Build {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
+		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
+			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+	before := record("c1aaaaaaaaaa", old)
+	record("c2bbbbbbbbbb", now)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
+		Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
+		t.Fatal(err)
+	}
+	// The mesh's record goes back to the build before, as the record went on 2026-10-02.
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+	wasMoves := machineMoves
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	t.Cleanup(func() { machineMoves = wasMoves })
+
+	judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
+	if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) ||
+		!strings.Contains(err.Error(), "mesh-controller c1aaaaaa") {
+		t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err)
+	}
+	putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})
+	if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
+		t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"register it, so it was not made.", module, m),
 			Resolved: "Resolved: " + module + " on " + m + " is settled"}
 	}),
+	// A send that would have put the controller or the node-engine back behind what a machine runs (novox/hq
+	// issue 201).
+	kindCoreDowngrade: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		component := idPart(o, 0)
+		if o.Scope != conditions.ScopeCore || !guarded(component) {
+			component = "core component"
+		}
+		return words{Headline: "An older " + component + " was not sent to " + m,
+			Needs: "decide which build of the " + component + " the mesh should hold; the details say which.",
+			Explanation: fmt.Sprintf("A send would have put the mesh's own %s on %s back to an older build than "+
+				"the one it runs, so it was not sent and %s keeps its build.", component, m, m),
+			Resolved: "Resolved: sends to " + m + " carry no older " + component}
+	}),
 	kindBindingKept: worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		return words{Headline: "A module's data source is held on " + m,

cmd/mesh-controller/release.go

@@ -51,6 +51,9 @@ type sendScope struct {
 	modules map[string]bool
 	// person is a person's act: it carries what it carries.
 	person bool
+	// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
+	// older than the one its machine runs (novox/hq issue 201).
+	puttingBack bool
 }
 
 type sendScopeKey struct{}
@@ -248,9 +251,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 	if err != nil {
 		return nil, err
 	}
-	var kept, refused []string
+	var kept, refused, older []string
 	for _, n := range names {
-		moves, err := machineMoves(ctx, open, f, n, false)
+		// Every move, gated or not: a move to a build that passed a gate elsewhere is no gate's concern here, and
+		// may still be a core module's downgrade (novox/hq issue 201).
+		every, err := machineMoves(ctx, open, f, n, true)
 		if err != nil {
 			return nil, err
 		}
@@ -262,11 +267,33 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 				return nil, err
 			}
 		}
-		var waiting []string
-		for _, mv := range moves {
+		var made []inventory.CarriedMove
+		for _, mv := range every {
 			if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
 				continue
 			}
+			made = append(made, mv)
+		}
+		// **No send but a gate's own put-back moves the controller or the node-engine to an older build than
+		// the one its machine runs** (novox/hq issue 201).
+		downgrades, err := refuseCoreDowngrades(ctx, open.inventory, scope, n, made)
+		if err != nil {
+			return nil, err
+		}
+		if len(downgrades) > 0 {
+			if n == addedHolder {
+				fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s\n", n,
+					strings.Join(downgrades, "; "))
+				continue
+			}
+			older = append(older, downgrades...)
+			continue
+		}
+		var waiting []string
+		for _, mv := range made {
+			if f.gated(mv.Module, mv.To) {
+				continue
+			}
 			if !scope.judged[n] && !scope.modules[mv.Module] {
 				waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
 			}
@@ -281,6 +308,9 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 			refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
 		}
 	}
+	if len(older) > 0 {
+		return nil, fmt.Errorf("%w: %s", errCoreDowngrade, strings.Join(older, "; "))
+	}
 	if len(refused) > 0 {
 		return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
 			"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))