Merge pull request 'A control plane behind its seat's row serves what it can (hq ADR 0185)' (#222) from fix/a-service-asked-to-run-is-still-running into main

cmd/mesh-controller/push.go

@@ -131,10 +131,17 @@ func serve(ctx context.Context) error {
 
 	// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
 	// from the store's row, so what the seat declares is what is answered.
-	handlers, err := seatToolHandlers()
+	handlers, behind, err := seatToolHandlers()
 	if err != nil {
 		return err
 	}
+	if len(behind) > 0 {
+		// Said once, loudly, and then served anyway (novox/hq ADR 0185): the mesh keeps answering
+		// while whatever put an older control plane here is undone.
+		fmt.Printf("this control plane is behind the %s row: it cannot run %s. "+
+			"Those answer the reason when called; everything else is served as usual\n",
+			catalogue.ControllerSeatName, strings.Join(behind, ", "))
+	}
 	bus, isNATS := server.Bus().(link.OverNATS)
 	if !isNATS {
 		return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")

cmd/mesh-controller/seatverbs.go

@@ -230,13 +230,15 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
 }
 
 // seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
-// store's row, so a verb the row does not carry is not served and a verb it carries that this binary
-// cannot run is said at start rather than at the first call.
-func seatToolHandlers() (map[string]link.ToolHandler, error) {
+// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
+// cannot run is named at start and answers the reason when called — never a refusal to serve, which
+// would take the whole control plane down for one word (novox/hq ADR 0185).
+func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 	seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
 	if !known {
-		return nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
+		return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
 	}
+	var behind []string
 	handlers := map[string]link.ToolHandler{}
 	for _, v := range seat.Serves {
 		verb := v.Name
@@ -247,8 +249,27 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) {
 			continue
 		}
 		if _, err := argvFor(verb, sampleArguments(v)); err != nil {
-			return nil, fmt.Errorf("the %s seat's row declares %q, which this control plane cannot run: %w",
-				catalogue.ControllerSeatName, verb, err)
+			// **A row ahead of this binary is not a reason to go silent.**
+			//
+			// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
+			// this build does not know means the row was widened by a newer one — the ordinary
+			// state of a roll-out, and of a push that put an older control plane back. Refusing to
+			// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
+			// mesh off the bus for ten minutes, and the way back was a human running the binary by
+			// hand, because the thing that would have repaired it is the thing that was down
+			// (novox/hq 04-ISSUES/201, ADR 0185).
+			//
+			// So the verbs this binary knows are served, and this one answers the reason instead of
+			// nothing: a caller gets a sentence naming the fault, and everything else keeps working
+			// — including the push that replaces this binary with the one whose verb it is.
+			behind = append(behind, verb)
+			reason := err
+			handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
+				return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
+					"here cannot run it: %w. It is a verb of a newer build; this one is behind",
+					verb, catalogue.ControllerSeatName, reason)
+			}
+			continue
 		}
 		handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
 			args := map[string]any{}
@@ -264,7 +285,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) {
 			return runVerb(ctx, argv)
 		}
 	}
-	return handlers, nil
+	return handlers, behind, nil
 }
 
 // seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the

cmd/mesh-controller/seatverbs_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"strings"
 	"testing"
 
@@ -130,10 +131,13 @@ func TestActsDoNotBlockTheCall(t *testing.T) {
 
 // What `tools` answers is the seats' records, with each verb's schema.
 func TestToolsAnswersTheSeatsRecords(t *testing.T) {
-	handlers, err := seatToolHandlers()
+	handlers, behind, err := seatToolHandlers()
 	if err != nil {
 		t.Fatal(err)
 	}
+	if len(behind) != 0 {
+		t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
+	}
 	if len(handlers) != len(catalogue.ControllerVerbs) {
 		t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
 	}
@@ -195,3 +199,53 @@ func TestCommandRunsTheLineAsGiven(t *testing.T) {
 		t.Fatal("an unclosed quote was accepted")
 	}
 }
+
+// A verb in the row that this binary cannot run does not take the control plane off the bus: the
+// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
+// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
+// a person running the binary by hand — the push that would have repaired it needs the control
+// plane that was down.
+func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
+	seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
+	if !known {
+		t.Fatal("no controller seat")
+	}
+	// The row as a newer control plane would have written it: every verb this build knows, and one
+	// it does not.
+	widened := seat
+	widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
+		catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
+	rows := catalogue.DefaultSeats()
+	for i := range rows {
+		if rows[i].Name == catalogue.ControllerSeatName {
+			rows[i] = widened
+		}
+	}
+	catalogue.UseSeats(rows)
+	t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
+
+	handlers, behind, err := seatToolHandlers()
+	if err != nil {
+		t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
+	}
+	if len(behind) != 1 || behind[0] != "teleport" {
+		t.Fatalf("the verbs this build cannot run were reported as %v", behind)
+	}
+	if len(handlers) != len(widened.Serves) {
+		t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
+	}
+	for _, known := range []string{"status", "nodes", "push"} {
+		if handlers[known] == nil {
+			t.Errorf("%s is not served although this build knows it", known)
+		}
+	}
+	_, err = handlers["teleport"](context.Background(), nil)
+	if err == nil {
+		t.Fatal("the unknown verb answered as though it had run")
+	}
+	for _, want := range []string{"teleport", "cannot run it", "behind"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the answer does not say %q: %v", want, err)
+		}
+	}
+}