diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 835525e..2a1ba11 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -131,10 +131,17 @@ func serve(ctx context.Context) error { // And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served // from the store's row, so what the seat declares is what is answered. - handlers, err := seatToolHandlers() + handlers, behind, err := seatToolHandlers() if err != nil { return err } + if len(behind) > 0 { + // Said once, loudly, and then served anyway (novox/hq ADR 0185): the mesh keeps answering + // while whatever put an older control plane here is undone. + fmt.Printf("this control plane is behind the %s row: it cannot run %s. "+ + "Those answer the reason when called; everything else is served as usual\n", + catalogue.ControllerSeatName, strings.Join(behind, ", ")) + } bus, isNATS := server.Bus().(link.OverNATS) if !isNATS { return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it") diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index a7d99ef..85d3fe3 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -230,13 +230,15 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) { } // seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the -// store's row, so a verb the row does not carry is not served and a verb it carries that this binary -// cannot run is said at start rather than at the first call. -func seatToolHandlers() (map[string]link.ToolHandler, error) { +// store's row, so a verb the row does not carry is not served. A verb it carries that this binary +// cannot run is named at start and answers the reason when called — never a refusal to serve, which +// would take the whole control plane down for one word (novox/hq ADR 0185). +func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) if !known { - return nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName) + return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName) } + var behind []string handlers := map[string]link.ToolHandler{} for _, v := range seat.Serves { verb := v.Name @@ -247,8 +249,27 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) { continue } if _, err := argvFor(verb, sampleArguments(v)); err != nil { - return nil, fmt.Errorf("the %s seat's row declares %q, which this control plane cannot run: %w", - catalogue.ControllerSeatName, verb, err) + // **A row ahead of this binary is not a reason to go silent.** + // + // The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb + // this build does not know means the row was widened by a newer one — the ordinary + // state of a roll-out, and of a push that put an older control plane back. Refusing to + // serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole + // mesh off the bus for ten minutes, and the way back was a human running the binary by + // hand, because the thing that would have repaired it is the thing that was down + // (novox/hq 04-ISSUES/201, ADR 0185). + // + // So the verbs this binary knows are served, and this one answers the reason instead of + // nothing: a caller gets a sentence naming the fault, and everything else keeps working + // — including the push that replaces this binary with the one whose verb it is. + behind = append(behind, verb) + reason := err + handlers[verb] = func(context.Context, json.RawMessage) (any, error) { + return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+ + "here cannot run it: %w. It is a verb of a newer build; this one is behind", + verb, catalogue.ControllerSeatName, reason) + } + continue } handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) { args := map[string]any{} @@ -264,7 +285,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, error) { return runVerb(ctx, argv) } } - return handlers, nil + return handlers, behind, nil } // seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index a072ec1..95cb8ba 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "strings" "testing" @@ -130,10 +131,13 @@ func TestActsDoNotBlockTheCall(t *testing.T) { // What `tools` answers is the seats' records, with each verb's schema. func TestToolsAnswersTheSeatsRecords(t *testing.T) { - handlers, err := seatToolHandlers() + handlers, behind, err := seatToolHandlers() if err != nil { t.Fatal(err) } + if len(behind) != 0 { + t.Fatalf("this build cannot run %v of its own seat's verbs", behind) + } if len(handlers) != len(catalogue.ControllerVerbs) { t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs)) } @@ -195,3 +199,53 @@ func TestCommandRunsTheLineAsGiven(t *testing.T) { t.Fatal("an unclosed quote was accepted") } } + +// A verb in the row that this binary cannot run does not take the control plane off the bus: the +// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR +// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by +// a person running the binary by hand — the push that would have repaired it needs the control +// plane that was down. +func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) { + seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) + if !known { + t.Fatal("no controller seat") + } + // The row as a newer control plane would have written it: every verb this build knows, and one + // it does not. + widened := seat + widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...), + catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"}) + rows := catalogue.DefaultSeats() + for i := range rows { + if rows[i].Name == catalogue.ControllerSeatName { + rows[i] = widened + } + } + catalogue.UseSeats(rows) + t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) + + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatalf("a row with one unknown verb refused to serve at all: %v", err) + } + if len(behind) != 1 || behind[0] != "teleport" { + t.Fatalf("the verbs this build cannot run were reported as %v", behind) + } + if len(handlers) != len(widened.Serves) { + t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves)) + } + for _, known := range []string{"status", "nodes", "push"} { + if handlers[known] == nil { + t.Errorf("%s is not served although this build knows it", known) + } + } + _, err = handlers["teleport"](context.Background(), nil) + if err == nil { + t.Fatal("the unknown verb answered as though it had run") + } + for _, want := range []string{"teleport", "cannot run it", "behind"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the answer does not say %q: %v", want, err) + } + } +}