node hand-over: the terminal hands a directory used as found to the mesh, asked of the node's engine (hq issue 356)

A module's condition told the operator to run the node-engine's hand-over at the machine, as root (issue 339), and the mesh had no channel for it. Now node hand-over <node> <path> is the controller's terminal's — a node subcommand that is not a read, so every verb and mesh-cli outside the terminal refuse it — and asks that node's engine on mesh.node.<node>.ask.hand-over, a request only the controller may publish and only that node's engine may hear and answer (its grant gains the subject and the right to answer what it was asked). The line's node and path are judged before anything is asked; the engine's answer is printed, a refusal as a refusal. Every text addressed to the operator names the nox line (ADR 0272); the condition's words stay plain.

internal/link/handover.go

@@ -0,0 +1,92 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// A hand-over asked of a machine's node-engine (novox/hq issue 356, issue 339).
+//
+// The operator hands a directory the node-engine uses as found to the mesh at the controller's terminal:
+// `nox node hand-over <node> <path>` on the control-node (ADR 0272). The controller asks that machine's
+// engine on its own subject, a request on core NATS the engine answers once; the engine judges every
+// value and records the hand-over, or refuses and records nothing. The engine holds the same two shapes
+// in its own link code (mesh-host internal/link HandOverAsk, HandOverAnswer); a test on each side holds
+// the field names.
+
+// HandOverAsk is what the controller asks: the directory's absolute path as the engine states it, and who
+// asked, in the controller's words.
+type HandOverAsk struct {
+	Path string `json:"path"`
+	By   string `json:"by"`
+}
+
+// HandOverAnswer is the engine's answer: what it recorded, or why it refused.
+type HandOverAnswer struct {
+	Said    string `json:"said,omitempty"`
+	Refused string `json:"refused,omitempty"`
+}
+
+// HandOverWithin is how long the controller waits for the engine's answer: a file write, on a machine that is
+// up; a machine that is down is said as not answering.
+const HandOverWithin = 30 * time.Second
+
+// AskHandOver asks one machine's node-engine to hand a directory used as found to the mesh, and reads its
+// answer. An error is the ask not reaching an engine, or an answer that is not one; a refusal is the engine's
+// and comes back in the answer.
+func AskHandOver(ctx context.Context, conn *nats.Conn, node, path, by string, timeout time.Duration) (HandOverAnswer, error) {
+	if conn == nil {
+		return HandOverAnswer{}, errors.New("this controller is not on the bus")
+	}
+	body, err := json.Marshal(HandOverAsk{Path: path, By: by})
+	if err != nil {
+		return HandOverAnswer{}, err
+	}
+	asking, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	subject := broker.AskHandOverSubject(node)
+	refused, stop := refusalsOf(conn, subject)
+	defer stop()
+	type replied struct {
+		msg *nats.Msg
+		err error
+	}
+	done := make(chan replied, 1)
+	go func() {
+		msg, err := conn.RequestWithContext(asking, subject, body)
+		done <- replied{msg, err}
+	}()
+	var reply *nats.Msg
+	select {
+	case r := <-done:
+		reply, err = r.msg, r.err
+	case why := <-refused:
+		cancel()
+		return HandOverAnswer{}, fmt.Errorf("the bus refused the controller asking %s for a hand-over: %v", node, why)
+	}
+	switch {
+	case errors.Is(err, nats.ErrNoResponders):
+		return HandOverAnswer{}, fmt.Errorf("nothing on %s answers a hand-over: its node-engine is not running, is not "+
+			"on the bus, or is older than this ask (novox/hq issue 356); nothing was handed over", node)
+	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
+		return HandOverAnswer{}, fmt.Errorf("%s did not answer the hand-over within %s; whether it was recorded is not "+
+			"known — the module's condition says whether the directory is still used as found", node, timeout)
+	case err != nil:
+		return HandOverAnswer{}, err
+	}
+	var answer HandOverAnswer
+	if err := json.Unmarshal(reply.Data, &answer); err != nil {
+		return HandOverAnswer{}, fmt.Errorf("%s answered the hand-over with something unreadable: %w", node, err)
+	}
+	if answer.Said == "" && answer.Refused == "" {
+		return HandOverAnswer{}, fmt.Errorf("%s answered the hand-over with neither a record nor a refusal", node)
+	}
+	return answer, nil
+}

internal/link/handover_test.go

@@ -0,0 +1,103 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// The hand-over's ask and answer hold these field names; the engine's side holds the same list (mesh-host
+// internal/link, TestTheHandOverAskAndAnswerKeepTheirFieldNames).
+func TestTheHandOverAskAndAnswerKeepTheirFieldNames(t *testing.T) {
+	body, _ := json.Marshal(HandOverAsk{Path: "/srv/notes", By: "jo through mesh-cli on anchor"})
+	if got := keysIn(t, body); got != "by path" {
+		t.Fatalf("the ask's fields are %q", got)
+	}
+	body, _ = json.Marshal(HandOverAnswer{Said: "s", Refused: "r"})
+	if got := keysIn(t, body); got != "refused said" {
+		t.Fatalf("the answer's fields are %q", got)
+	}
+	if broker.AskHandOverSubject("laptop") != "mesh.node.laptop.ask.hand-over" {
+		t.Fatalf("the subject is %q", broker.AskHandOverSubject("laptop"))
+	}
+}
+
+// The ask reaches the machine's engine on its own subject and its answer comes back whole: what it recorded, or
+// its refusal as the engine worded it. A machine with no engine listening is said as not answering, and an
+// answer that is neither is refused rather than read as a record.
+func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) {
+	url := testbus.URL(t)
+	conn, err := nats.Connect(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(conn.Close)
+
+	var heard HandOverAsk
+	engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) {
+		_ = json.Unmarshal(msg.Data, &heard)
+		switch heard.Path {
+		case "/srv/notes":
+			body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By})
+			_ = msg.Respond(body)
+		case "/srv/empty":
+			_ = msg.Respond([]byte(`{}`))
+		default:
+			body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"})
+			_ = msg.Respond(body)
+		}
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = engine.Unsubscribe() })
+
+	ctx := context.Background()
+	a, err := AskHandOver(ctx, conn, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second)
+	if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") {
+		t.Fatalf("answered %+v, %v", a, err)
+	}
+	if heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" {
+		t.Fatalf("the engine heard %+v", heard)
+	}
+	a, err = AskHandOver(ctx, conn, "laptop", "/srv/other", "jo", 5*time.Second)
+	if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") {
+		t.Fatalf("a refusal came back as %+v, %v", a, err)
+	}
+	if _, err := AskHandOver(ctx, conn, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil ||
+		!strings.Contains(err.Error(), "neither") {
+		t.Fatalf("an empty answer was taken: %v", err)
+	}
+	if _, err := AskHandOver(ctx, conn, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil ||
+		!strings.Contains(err.Error(), "node-engine") {
+		t.Fatalf("a machine with no engine listening: %v", err)
+	}
+	if _, err := AskHandOver(ctx, nil, "anchor", "/srv/notes", "jo", time.Second); err == nil {
+		t.Fatal("asked with no bus")
+	}
+}
+
+// A machine's grant hears its own hand-over ask and nobody else's, and may answer it: the one request a node is
+// asked (novox/hq issue 356).
+func TestAMachineHearsItsOwnHandOverAskAndMayAnswerIt(t *testing.T) {
+	perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var hears, other bool
+	for _, s := range perms.Subscribe {
+		hears = hears || s == "mesh.node.laptop.ask.hand-over"
+		other = other || s == "mesh.node.anchor.ask.hand-over"
+	}
+	if !hears || other || !perms.AllowResponses {
+		t.Fatalf("a machine's grant: hears its own %v, another's %v, answers %v (%v)", hears, other, perms.AllowResponses,
+			perms.Subscribe)
+	}
+}

internal/link/protocol.go

@@ -471,8 +471,9 @@ const KindUnit = "unit"
 const KindAccount = "account"
 
 // KindDirectory is a directory of a module that the node-engine uses as found (novox/hq issue 339): there before
-// the mesh, with another owner or mode than declared, and left so until a person hands it over at the machine
-// (`mesh-host hand-over`). Stated unhealthy with a reason that starts ReasonUsedAsFound.
+// the mesh, with another owner or mode than declared, and left so until the operator hands it over at the
+// controller's terminal (`nox node hand-over <node> <path>`, issue 356). Stated unhealthy with a reason that starts
+// ReasonUsedAsFound.
 const KindDirectory = "directory"
 
 // ReasonUsedAsFound starts the reason of a directory used as found.