cmd/mesh-controller/core_downgrade.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// **A send never moves the mesh's own core to a build older than the one its machine runs** (novox/hq issue
+// 201). On 2026-10-02 two merges a minute apart made two plans; the controller's plan rolled the new controller,
+// which widened its seat's row, and the node-engine's plan then sent the control node the controller build
+// before it. The older binary read the row it could not serve and the mesh lost its voice for ten minutes — the
+// thing that would have corrected it was the thing that was down. ADR 0185 took the outage out of it (a
+// controller behind its row serves what it can); the race is closed here, without needing its trigger: whatever
+// composed the older build, the send that would carry it is refused.
+//
+// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the
+// commits' dates. A gate's own put-back is the one send that may carry it, for the modules it puts back: it says
+// so (sendScope.puttingBack), and the controller's put-back is held to the store's schema already (issue 352).
+// A person's push carries what it carries, as every other refusal of a send leaves it.
+//
+// The refusal is said in the plan's words — the send's error, which the plan keeps as its note and tries again
+// — and as a warning condition on the machine, cleared by the first send there that carries no such move.
+
+// errCoreDowngrade is a send refused because it would put a core module on a machine back behind the build that
+// machine runs.
+var errCoreDowngrade = errors.New("a send would put the mesh's own core back behind the build its machine runs")
+
+// kindCoreDowngrade is that refusal, as a condition.
+const kindCoreDowngrade = "core-downgrade-refused"
+
+// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
+const coreDowngradeSource = "sends"
+
+// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
+// store and its seat's row, and the node-engine, which reads what the controller declares.
+var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
+
+// buildAsked is when the newest build of a module from a commit that worked was asked — or recorded, when the
+// request time is not known — and false when the records hold none. A variable, so a test reads its own order.
+var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (time.Time, bool, error) {
+	builds, err := inv.Builds(ctx, module, askedLookBack)
+	if err != nil {
+		return time.Time{}, false, err
+	}
+	for _, b := range builds {
+		if b.Worked() && sameCommit(b.Commit, commit) {
+			return b.AskedOrAt(), true, nil
+		}
+	}
+	return time.Time{}, false, nil
+}
+
+// askedLookBack is how many of a module's newest builds are read for when a build was asked.
+const askedLookBack = 50
+
+// coreDowngrade is one move of a core module to an older build than its machine runs.
+type coreDowngrade struct {
+	move             inventory.CarriedMove
+	component        string
+	fromAsked, asked time.Time
+}
+
+func (d coreDowngrade) String() string {
+	return fmt.Sprintf("%s would be sent %s %s, older than the %s it runs (asked %s, before %s) — no send but a "+
+		"gate's own put-back moves the %s back (novox/hq issue 201)", d.move.Node, d.move.Module, short(d.move.To),
+		short(d.move.From), d.asked.UTC().Format(time.RFC3339), d.fromAsked.UTC().Format(time.RFC3339), d.component)
+}
+
+// coreDowngrades is the moves of a machine's send that put a core module back to a build asked before the one it
+// runs, leaving out what a put-back puts back. A move whose either build the records do not hold is not proved
+// older, and is left to the gate that judges it. Pure but for asked.
+func coreDowngrades(moves []inventory.CarriedMove, scope sendScope,
+	asked func(module, commit string) (time.Time, bool, error)) ([]coreDowngrade, error) {
+	var out []coreDowngrade
+	for _, mv := range moves {
+		component := coreComponent(mv.Module)
+		if !guarded(component) || mv.From == "" || mv.To == "" || sameCommit(mv.From, mv.To) {
+			continue
+		}
+		if scope.puttingBack && scope.modules[mv.Module] {
+			continue
+		}
+		from, known, err := asked(mv.Module, mv.From)
+		if err != nil || !known {
+			if err != nil {
+				return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.From), err)
+			}
+			continue
+		}
+		to, known, err := asked(mv.Module, mv.To)
+		if err != nil {
+			return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", mv.Module, short(mv.To), err)
+		}
+		if known && to.Before(from) {
+			out = append(out, coreDowngrade{move: mv, component: component, fromAsked: from, asked: to})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].move.Module < out[j].move.Module })
+	return out, nil
+}
+
+func guarded(component string) bool {
+	for _, c := range downgradeGuarded {
+		if c == component {
+			return true
+		}
+	}
+	return false
+}
+
+// refuseCoreDowngrades says, for one machine's moves, each that would put a core module back, raising its
+// condition; and clears the condition of each guarded component that machine's send does not move back.
+func refuseCoreDowngrades(ctx context.Context, inv *inventory.Inventory, scope sendScope, node string,
+	moves []inventory.CarriedMove) ([]string, error) {
+	found, err := coreDowngrades(moves, scope, func(module, commit string) (time.Time, bool, error) {
+		return buildAsked(ctx, inv, module, commit)
+	})
+	if err != nil {
+		return nil, err
+	}
+	var said []string
+	refused := map[string]bool{}
+	for _, d := range found {
+		said = append(said, d.String())
+		refused[d.component] = true
+		sayCoreDowngrade(ctx, d)
+	}
+	for _, c := range downgradeGuarded {
+		if !refused[c] {
+			clearCoreDowngrade(ctx, c, node)
+		}
+	}
+	return said, nil
+}
+
+// coreDowngradeObservation is a refused downgrade as a condition: `core.<component>.<machine>.<kind>`.
+func coreDowngradeObservation(component, node, summary string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
+		Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
+		Summary: summary}
+}
+
+func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
+	fmt.Printf("NOT SENT: %s\n", d)
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.move.Node, d.String())); err != nil {
+		fmt.Printf("the condition for %s's refused downgrade on %s could not be raised: %v\n", d.move.Module,
+			d.move.Node, err)
+	}
+}
+
+func clearCoreDowngrade(ctx context.Context, component, node string) {
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	key := coreDowngradeObservation(component, node, "").Key()
+	if _, err := keeper.Clear(ctx, key, "a send to "+node+" carries no older "+component); err != nil {
+		fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
+	}
+}

cmd/mesh-controller/gate.go

@@ -1039,7 +1039,7 @@ func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module str
 		}
 		return
 	}
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}}), open, g.Machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines)
 	if err != nil {
 		notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+
 			"sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0]))
@@ -1132,7 +1132,7 @@ func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *roll
 	}
 	inv := open.inventory
 	sort.Strings(b.machines)
-	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules}), open, b.machines)
+	sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines)
 	var back []string
 	for _, r := range b.pending {
 		if err != nil {

cmd/mesh-controller/issue201_test.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+)
+
+// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there,
+// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for
+// ten minutes. No send but a gate's own put-back moves the controller or the node-engine back behind the build its
+// machine runs, by the builds' registration order.
+
+// askedIn answers when a build was asked from a fixed table: module@commit → when.
+func askedIn(table map[string]time.Time) func(module, commit string) (time.Time, bool, error) {
+	return func(module, commit string) (time.Time, bool, error) {
+		at, known := table[module+"@"+commit]
+		return at, known, nil
+	}
+}
+
+func TestACoreModuleIsNeverMovedBehindTheBuildItsMachineRuns(t *testing.T) {
+	old, now := t0, t0.Add(time.Minute)
+	asked := askedIn(map[string]time.Time{
+		"mesh-controller@c1": old, "mesh-controller@c2": now,
+		"mesh-host@h1": old, "mesh-host@h2": now,
+		"app@a1": old, "app@a2": now,
+	})
+	back := []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"},
+		{Module: "mesh-host", Node: "anchor", From: "h2", To: "h1"},
+		{Module: "app", Node: "anchor", From: "a2", To: "a1"},
+	}
+	found, err := coreDowngrades(back, sendScope{judged: map[string]bool{"anchor": true}}, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 2 || found[0].move.Module != "mesh-controller" || found[0].component != lease.ComponentController ||
+		found[1].move.Module != "mesh-host" || found[1].component != lease.ComponentEngine {
+		t.Fatalf("the controller and the node-engine sent back, and nothing else, are refused: %+v", found)
+	}
+	if said := found[0].String(); !strings.Contains(said, "anchor would be sent mesh-controller c1") ||
+		!strings.Contains(said, "issue 201") {
+		t.Fatalf("the refusal says the machine, the module and the build: %s", said)
+	}
+
+	// Forward, the same build, or a build the records do not hold: not refused.
+	for _, mv := range []inventory.CarriedMove{
+		{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c2"},
+		{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c9"},
+		{Module: "mesh-controller", Node: "anchor", From: "c9", To: "c1"},
+	} {
+		if found, err := coreDowngrades([]inventory.CarriedMove{mv}, sendScope{}, asked); err != nil || len(found) != 0 {
+			t.Fatalf("%+v is no downgrade proved: %+v %v", mv, found, err)
+		}
+	}
+
+	// A gate's own put-back carries its own modules back, and nothing else.
+	putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true}
+	found, err = coreDowngrades(back, putBack, asked)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(found) != 1 || found[0].move.Module != "mesh-host" {
+		t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found)
+	}
+	// Naming a module is not a put-back.
+	if found, _ := coreDowngrades(back[:1], sendScope{modules: map[string]bool{"mesh-controller": true}}, asked); len(found) != 1 {
+		t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found)
+	}
+}
+
+// The refusal is a warning condition on the machine, which the next send there carrying no such move clears.
+func TestARefusedCoreDowngradeIsSaidAndClearedByTheNextSend(t *testing.T) {
+	k, _ := withConditionsInMemory(t)
+	ctx := t.Context()
+	was := buildAsked
+	buildAsked = func(_ context.Context, _ *inventory.Inventory, module, commit string) (time.Time, bool, error) {
+		return askedIn(map[string]time.Time{"mesh-controller@c1": t0, "mesh-controller@c2": t0.Add(time.Minute)})(module, commit)
+	}
+	t.Cleanup(func() { buildAsked = was })
+
+	said, err := refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c2", To: "c1"}})
+	if err != nil || len(said) != 1 {
+		t.Fatalf("the downgrade is refused: %v %v", said, err)
+	}
+	key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key()
+	c, open, err := k.Get(ctx, key)
+	if err != nil || !open || !strings.Contains(c.Summary, "mesh-controller c1") {
+		t.Fatalf("the condition %s is raised, saying the build: %+v %v %v", key, c, open, err)
+	}
+
+	said, err = refuseCoreDowngrades(ctx, nil, sendScope{}, "anchor",
+		[]inventory.CarriedMove{{Module: "mesh-controller", Node: "anchor", From: "c1", To: "c2"}})
+	if err != nil || len(said) != 0 {
+		t.Fatalf("a move forward is sent: %v %v", said, err)
+	}
+	if _, open, err := k.Get(ctx, key); err != nil || open {
+		t.Fatalf("the condition is cleared by a send carrying no older controller: %v %v", open, err)
+	}
+}
+
+// TestReplay201 replays the race on a store: the control node runs the newer controller, the mesh's record of
+// the controller is the build before it, and a plan's gated send there — the node-engine's plan of 2026-10-02 —
+// is refused, said as the plan's words; the gate's own put-back of the controller is not.
+func TestReplay201(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	record := func(commit string, asked time.Time) inventory.Build {
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
+		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
+			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+	before := record("c1aaaaaaaaaa", old)
+	record("c2bbbbbbbbbb", now)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
+		Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil {
+		t.Fatal(err)
+	}
+	// The mesh's record goes back to the build before, as the record went on 2026-10-02.
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+	wasMoves := machineMoves
+	machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
+		modules, err := open.inventory.Assigned(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		sent, known, err := open.inventory.SentBuilds(ctx, node)
+		if err != nil {
+			return nil, err
+		}
+		return f.moves(node, modules, sent, known, all), nil
+	}
+	t.Cleanup(func() { machineMoves = wasMoves })
+
+	judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}})
+	if _, err := ungatedIn(judged, open, []string{"anchor"}, ""); !errors.Is(err, errCoreDowngrade) ||
+		!strings.Contains(err.Error(), "mesh-controller c1aaaaaa") {
+		t.Fatalf("a plan's send of the older controller to the machine running the newer is refused: %v", err)
+	}
+	putBack := withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})
+	if kept, err := ungatedIn(putBack, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 {
+		t.Fatalf("the gate's own put-back of the controller is sent: %v %v", kept, err)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -182,6 +182,20 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"register it, so it was not made.", module, m),
 			Resolved: "Resolved: " + module + " on " + m + " is settled"}
 	}),
+	// A send that would have put the controller or the node-engine back behind what a machine runs (novox/hq
+	// issue 201).
+	kindCoreDowngrade: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		component := idPart(o, 0)
+		if o.Scope != conditions.ScopeCore || !guarded(component) {
+			component = "core component"
+		}
+		return words{Headline: "An older " + component + " was not sent to " + m,
+			Needs: "decide which build of the " + component + " the mesh should hold; the details say which.",
+			Explanation: fmt.Sprintf("A send would have put the mesh's own %s on %s back to an older build than "+
+				"the one it runs, so it was not sent and %s keeps its build.", component, m, m),
+			Resolved: "Resolved: sends to " + m + " carry no older " + component}
+	}),
 	kindBindingKept: worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		return words{Headline: "A module's data source is held on " + m,

cmd/mesh-controller/release.go

@@ -51,6 +51,9 @@ type sendScope struct {
 	modules map[string]bool
 	// person is a person's act: it carries what it carries.
 	person bool
+	// puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build
+	// older than the one its machine runs (novox/hq issue 201).
+	puttingBack bool
 }
 
 type sendScopeKey struct{}
@@ -248,9 +251,11 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 	if err != nil {
 		return nil, err
 	}
-	var kept, refused []string
+	var kept, refused, older []string
 	for _, n := range names {
-		moves, err := machineMoves(ctx, open, f, n, false)
+		// Every move, gated or not: a move to a build that passed a gate elsewhere is no gate's concern here, and
+		// may still be a core module's downgrade (novox/hq issue 201).
+		every, err := machineMoves(ctx, open, f, n, true)
 		if err != nil {
 			return nil, err
 		}
@@ -262,11 +267,33 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 				return nil, err
 			}
 		}
-		var waiting []string
-		for _, mv := range moves {
+		var made []inventory.CarriedMove
+		for _, mv := range every {
 			if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {
 				continue
 			}
+			made = append(made, mv)
+		}
+		// **No send but a gate's own put-back moves the controller or the node-engine to an older build than
+		// the one its machine runs** (novox/hq issue 201).
+		downgrades, err := refuseCoreDowngrades(ctx, open.inventory, scope, n, made)
+		if err != nil {
+			return nil, err
+		}
+		if len(downgrades) > 0 {
+			if n == addedHolder {
+				fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s\n", n,
+					strings.Join(downgrades, "; "))
+				continue
+			}
+			older = append(older, downgrades...)
+			continue
+		}
+		var waiting []string
+		for _, mv := range made {
+			if f.gated(mv.Module, mv.To) {
+				continue
+			}
 			if !scope.judged[n] && !scope.modules[mv.Module] {
 				waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To)))
 			}
@@ -281,6 +308,9 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 			refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", ")))
 		}
 	}
+	if len(older) > 0 {
+		return nil, fmt.Errorf("%w: %s", errCoreDowngrade, strings.Join(older, "; "))
+	}
 	if len(refused) > 0 {
 		return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
 			"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))