Run a verb from the controller's own running image, and refuse what it cannot run as a handover (hq issue 289)

The witness moves a running build aside into a directory the controller's user
cannot enter, then deletes it; a verb exec'd from os.Executable() in that window
failed with permission denied. /proc/self/exe stays valid while the process lives.
A verb that still cannot start, or arrives while the controller stops, is refused
with link.ErrHandingOver and marked retry: handing-over.

internal/link/calls.go

@@ -29,6 +29,15 @@ import (
 // then and with "still running as call <id>" when it does not; and every call is kept here, with
 // what came of it, including an answer the bus refused, so `calls` can say it.
 
+// ErrHandingOver is a call refused because the controller answering it is being replaced: stopping, or
+// unable to run its own build because the node-engine has moved it aside (novox/hq issue 289). Nothing
+// was done, and the controller after it answers the same call — so the answer carries
+// `"retry": RetryHandingOver`, and a caller asks once more.
+var ErrHandingOver = errors.New("the controller is handing over to the next one; nothing was done, ask again")
+
+// RetryHandingOver is the `retry` mark of an answer refused by ErrHandingOver.
+const RetryHandingOver = "handing-over"
+
 // AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
 // inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
 // own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
@@ -524,7 +533,10 @@ func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply strin
 		var body []byte
 		result, err := handle(ctx, args)
 		failed := err != nil
-		if err != nil {
+		if errors.Is(err, ErrHandingOver) {
+			// Marked as well as said, so a caller asks again without reading the words (issue 289).
+			body, _ = json.Marshal(map[string]any{"error": err.Error(), "retry": RetryHandingOver})
+		} else if err != nil {
 			body, _ = json.Marshal(map[string]any{"error": err.Error()})
 		} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
 			failed = true

internal/link/calls_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log"
 	"strings"
 	"sync"
@@ -177,3 +178,24 @@ func recentOf(l *CallLog) []Call {
 	out, _ := l.Recent()
 	return out
 }
+
+// A call refused because its controller is handing over says so in a mark as well as in words, so the
+// caller asks once more without parsing a sentence (novox/hq issue 289).
+func TestAHandoverRefusalIsMarkedRetryable(t *testing.T) {
+	l, a := NewCallLog(), newAnswers(t)
+	l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.9", func(context.Context, json.RawMessage) (any, error) {
+		return nil, fmt.Errorf("%w: stopping", ErrHandingOver)
+	}, a.respond, nil)
+	got := a.only()
+	if got["retry"] != RetryHandingOver || !strings.Contains(fmt.Sprint(got["error"]), "handing over") {
+		t.Fatalf("answered %v", got)
+	}
+
+	l, a = NewCallLog(), newAnswers(t)
+	l.serveCall("mesh-controller", "rotate", nil, "_INBOX.x.10", func(context.Context, json.RawMessage) (any, error) {
+		return nil, errors.New("refused for its own reason")
+	}, a.respond, nil)
+	if _, marked := a.only()["retry"]; marked {
+		t.Fatal("an ordinary refusal was marked to be asked again")
+	}
+}