Answer every seat call within ten seconds and keep what came of it (hq issue 265)

A push outlasted the console's 30s wait and, when it sent the bus its
changed user list, the broker's reload forgot the reply it may send:
the push happened and its caller was told it did not answer. Calls now
answer in full or as running with an id, a push answers before it
sends, refused answers are recorded on their call, and 'calls' reads
them back.

cmd/mesh-controller/calls_verb_test.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
+// RequestTimeout) — the shortest wait of a caller the mesh ships.
+const consoleWaits = 30 * time.Second
+
+// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
+// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
+// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
+func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
+	if link.AnswerWithin >= consoleWaits/2 {
+		t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
+	}
+	if link.AnswerWithin >= broker.ResponseTTL {
+		t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
+	}
+}
+
+// A push — named or through command — answers before it runs: it sends the machine holding the bus
+// first, and the broker reloading its user list forgets the answer it was about to permit.
+func TestAPushAnswersBeforeItSends(t *testing.T) {
+	for _, c := range []struct {
+		verb string
+		args map[string]any
+		want bool
+	}{
+		{"push", map[string]any{"node": "anchor"}, true},
+		{"push", map[string]any{}, true},
+		{"command", map[string]any{"command": "push anchor"}, true},
+		{"command", map[string]any{"command": "push --behind"}, true},
+		{"command", map[string]any{"command": "builds"}, false},
+		{"status", map[string]any{}, false},
+		{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
+	} {
+		argv, err := argvFor(c.verb, c.args)
+		if err != nil {
+			t.Fatalf("%s %v: %v", c.verb, c.args, err)
+		}
+		if got := answersFirst(argv); got != c.want {
+			t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
+		}
+	}
+}
+
+// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
+func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
+	handlers, behind, err := seatToolHandlers()
+	if err != nil || len(behind) != 0 {
+		t.Fatalf("%v %v", behind, err)
+	}
+	calls, ok := handlers["calls"]
+	if !ok {
+		t.Fatal("calls is not served")
+	}
+	if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
+		!strings.Contains(err.Error(), `"node"`) {
+		t.Errorf("calls took an argument it does not declare: %v", err)
+	}
+	if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
+		!strings.Contains(err.Error(), "not across a restart") {
+		t.Errorf("an unknown call was not said plainly: %v", err)
+	}
+	got, err := calls(context.Background(), json.RawMessage(`{}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, listed := got.(map[string]any)["calls"]; !listed {
+		t.Errorf("calls answered %v", got)
+	}
+}

cmd/mesh-controller/push.go

@@ -155,6 +155,8 @@ func serve(ctx context.Context) error {
 	if !isNATS {
 		return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
 	}
+	// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
+	link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
 	stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
 	if err != nil {
 		return err

cmd/mesh-controller/seatverbs.go

@@ -485,7 +485,7 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 	handlers := map[string]link.ToolHandler{}
 	for _, v := range seat.Serves {
 		verb := v.Name
-		if verb == "tools" {
+		if inProcess[verb] {
 			handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
 				args := map[string]any{}
 				if len(bytes.TrimSpace(raw)) > 0 {
@@ -493,10 +493,14 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 						return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
 					}
 				}
-				// Refused like any verb's: tools takes nothing, and something given is not ignored.
-				if _, err := readArguments(verb, args); err != nil {
+				// Refused like any verb's: what a verb does not take is not ignored.
+				a, err := readArguments(verb, args)
+				if err != nil {
 					return nil, err
 				}
+				if verb == "calls" {
+					return callsAnswer(link.Calls, a.given["call"])
+				}
 				return seatTools(), nil
 			}
 			continue
@@ -535,12 +539,48 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 			if err != nil {
 				return nil, err
 			}
+			if answersFirst(argv) {
+				// Before anything is sent: a push sends the bus's own machine first, and a broker
+				// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
+				link.Acknowledge(ctx)
+			}
 			return runVerb(ctx, argv)
 		}
 	}
 	return handlers, behind, nil
 }
 
+// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
+// the records, `calls` from what this process served.
+var inProcess = map[string]bool{"tools": true, "calls": true}
+
+// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
+// through `command`. A push sends the machine holding the bus first when its user list changed, the
+// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
+// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
+func answersFirst(argv []string) bool {
+	return len(argv) > 0 && argv[0] == "push"
+}
+
+// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
+// one call whole.
+func callsAnswer(log *link.CallLog, id string) (any, error) {
+	if id != "" {
+		c, ok := log.Get(id)
+		if !ok {
+			return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
+				"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
+		}
+		return c, nil
+	}
+	recent := log.Recent()
+	for i := range recent {
+		recent[i].Answer = nil
+	}
+	return map[string]any{"calls": recent, "kept": link.KeptCalls,
+		"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
+}
+
 // seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
 // mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
 func seatTools() map[string]any {

cmd/mesh-controller/seatverbs_schema_test.go

@@ -80,7 +80,7 @@ var saidOutright = map[string]string{
 // shape of the push that named a machine and pushed every machine behind.
 func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
 	for name, v := range servedSchemas(t) {
-		if name == "tools" {
+		if inProcess[name] {
 			continue
 		}
 		names, switches := declaredArguments(v)
@@ -123,7 +123,7 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
 			args := sampleArguments(v)
 			args[stranger] = "x"
 			_, err := argvFor(name, args)
-			if name == "tools" {
+			if inProcess[name] {
 				_, err = readArguments(name, args)
 			}
 			if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
@@ -283,7 +283,7 @@ func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
 	reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
 	served := servedSchemas(t)
 	for name, v := range served {
-		if name == "tools" || name == "command" {
+		if inProcess[name] || name == "command" {
 			continue
 		}
 		names, switches := declaredArguments(v)