A node is given how it hears its declarations as it enrols

The node's declaration consumer was asserted only when the control plane
started, so the first machine of a mesh, enrolling after the control
plane was up, joined and then heard nothing: its host retried 'consumer
not found' for ever (novox/hq issue 146).

cmd/mesh-controller/network.go

@@ -232,9 +232,22 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
 	if err != nil {
 		return nil, err
 	}
+	// **A machine joining is on the network before it is anything else** (novox/hq ADR 0169). Its
+	// token was issued for its tunnel key and gave it an address, so while that token can still be
+	// used the hub carries it as a peer: it brings its tunnel up from the token and enrols over it.
+	// When the token is spent the machine is on the network by what it runs, as every other is; when
+	// it expires unused, the peer goes with it at the hub's next composition.
+	joining, err := inv.NodesWithALiveToken(ctx)
+	if err != nil {
+		return nil, err
+	}
+	isJoining := map[string]bool{}
+	for _, name := range joining {
+		isJoining[name] = true
+	}
 	nodes := make([]overlay.Node, 0, len(places))
 	for _, p := range places {
-		if !on[p.Name] {
+		if !on[p.Name] && !(isJoining[p.Name] && p.Key != "" && p.Address != "") {
 			continue
 		}
 		n := overlay.Node{

cmd/mesh-controller/nodes.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"flag"
 	"fmt"
+	"net"
 	"strings"
 	"time"
 
@@ -230,6 +232,8 @@ func tokenCommand(ctx context.Context, args []string) error {
 	validFor := set.Duration("for", time.Hour, "how long the token may be used")
 	adopted := set.Bool("adopted", false,
 		"the machine joining is in use: it is adopted, and keeps what is found on it")
+	tunnelKey := set.String("overlay-key", "",
+		"the public half of the tunnel key the machine made (`nox-mesh-host key`): it joins through the tunnel")
 	if err := set.Parse(args[1:]); err != nil {
 		return err
 	}
@@ -283,6 +287,14 @@ func tokenCommand(ctx context.Context, args []string) error {
 	default:
 		return err
 	}
+	// **Through the tunnel** (novox/hq ADR 0169): the machine's key recorded, its address given, the
+	// hub sent it as a peer — all before the token is shown, so the tunnel answers the first time the
+	// machine knocks. The bus is then reached at its address on the private network.
+	if *tunnelKey != "" {
+		if made.Tunnel, made.Broker, err = throughTheTunnel(ctx, open, issued.Node, *tunnelKey, made.Broker); err != nil {
+			return err
+		}
+	}
 	encoded, err := made.Encode()
 	if err != nil {
 		return err
@@ -307,6 +319,66 @@ func tokenCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+// throughTheTunnel makes a machine a peer of the hub for its token, and says what the token carries
+// for it: its first tunnel, and the bus at its address on the private network (novox/hq ADR 0169).
+//
+// The hub is pushed here, before the token is shown. A token shown before the hub knew the key is a
+// tunnel that does not answer, and a machine that cannot tell that from a bus that is down.
+func throughTheTunnel(ctx context.Context, open *stores, node inventory.Node, key, busAt string) (
+	*token.Tunnel, string, error) {
+	inv := open.inventory
+	key = strings.TrimSpace(key)
+	if raw, err := base64.StdEncoding.DecodeString(key); err != nil || len(raw) != 32 {
+		return nil, "", fmt.Errorf("%q is not a tunnel public key: it is 32 bytes in base64, as "+
+			"`nox-mesh-host key` prints it", key)
+	}
+	// The bus on the private network is the hub's address at the bus's own port, so the port must be
+	// known before anything is recorded.
+	_, port, err := net.SplitHostPort(busAt)
+	if err != nil || port == "" {
+		return nil, "", fmt.Errorf("the bus's address %q has no port to reach it on", busAt)
+	}
+	places, err := inv.Overlays(ctx)
+	if err != nil {
+		return nil, "", err
+	}
+	var hub *inventory.Overlay
+	for i := range places {
+		if places[i].Hub {
+			hub = &places[i]
+		}
+	}
+	if hub == nil || hub.Key == "" || hub.Endpoint == "" || hub.Address == "" {
+		return nil, "", errors.New("this mesh has no hub with a key, an address and an endpoint to " +
+			"dial, so there is no tunnel to join through: place one (`overlay place <node> --hub " +
+			"--endpoint <host>:<port>`), or issue the token without --overlay-key")
+	}
+	if err := inv.RecordOverlayKey(ctx, node.ID, key); err != nil {
+		return nil, "", err
+	}
+	if err := inv.BindTokenToKey(ctx, node.ID, key); err != nil {
+		return nil, "", err
+	}
+	cidr, err := overlayRange(ctx, inv)
+	if err != nil {
+		return nil, "", err
+	}
+	address, err := inv.AssignAddress(ctx, node.ID, cidr)
+	if err != nil {
+		return nil, "", err
+	}
+	if err := sendTo(ctx, open, []string{hub.Name}); err != nil {
+		return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub could not be sent "+
+			"it, so the tunnel would not answer — the token is not shown; issue it again once %s "+
+			"can be pushed: %w", node.Name, hub.Name, err)
+	}
+	// An address, not a name — nothing resolves before the machine has joined (novox/hq ADR 0004).
+	return &token.Tunnel{
+		Key: key, Address: address + "/32", Range: cidr,
+		HubKey: hub.Key, HubEndpoint: hub.Endpoint,
+	}, net.JoinHostPort(hub.Address, port), nil
+}
+
 // issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
 // when the operator says so, and the one-time secret for it. The node in what it returns carries
 // its mode, which is what the token says.

cmd/mesh-controller/seatverbs.go

@@ -144,6 +144,26 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
 		// Half of either shape: the command says its usage, which names both shapes, and that is
 		// the answer the caller needs.
 		return []string{"rotate"}, nil
+	case "token":
+		// `token issue` at a shell (novox/hq ADR 0169). Exactly one of node or new; the command
+		// refuses both or neither in its own words.
+		argv := []string{"token", "issue"}
+		if n := str("node"); n != "" {
+			argv = append(argv, "--node", n)
+		}
+		if n := str("new"); n != "" {
+			argv = append(argv, "--new", n)
+		}
+		if k := str("overlay_key"); k != "" {
+			argv = append(argv, "--overlay-key", k)
+		}
+		if d := str("for"); d != "" {
+			argv = append(argv, "--for", d)
+		}
+		if str("adopted") == "true" {
+			argv = append(argv, "--adopted")
+		}
+		return argv, nil
 	case "settings":
 		// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
 		// because a tool has no file to hand the command; the command reads either.

cmd/mesh-controller/seatverbs_test.go

@@ -73,6 +73,14 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
 	}
 }
 
+// `token` is `token issue` at a shell, with the machine's tunnel key (novox/hq ADR 0169).
+func TestTokenIssuesForAMachineAndItsTunnelKey(t *testing.T) {
+	argv, err := argvFor("token", map[string]any{"new": "laptop", "overlay_key": "k", "for": "2h"})
+	if err != nil || strings.Join(argv, " ") != "token issue --new laptop --overlay-key k --for 2h" {
+		t.Fatalf("token: %v %v", argv, err)
+	}
+}
+
 // `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
 func TestSettingsSetsOrClearsALayer(t *testing.T) {
 	argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})

internal/catalogue/verbs.go

@@ -136,6 +136,16 @@ var ControllerVerbs = []Verb{
 			"node":   "the machine that runs the module",
 			"module": "the module's name",
 		}, []string{"node", "module"})},
+	{Name: "token", Description: "Issue a one-time token for a machine to join with. Give the public half of the " +
+		"tunnel key the machine made (`nox-mesh-host key`): the machine is given its address and made a peer of " +
+		"the hub, and joins through the tunnel. The token is shown once, in the answer.",
+		Input: schema(map[string]string{
+			"node":        "a machine the mesh already has a record for",
+			"new":         "or the name of a machine to create the record for",
+			"overlay_key": "the public half of the machine's tunnel key",
+			"for":         "how long it may be used, as a duration (default 1h)",
+			"adopted":     "\"true\" when the machine is in use and joins adopted",
+		}, nil)},
 	{Name: "settings", Description: "Set what an assignment is configured with: a module's settings for the whole mesh, " +
 		"or for one machine. Replaces that layer whole — what it does not name, it no longer sets — and takes effect " +
 		"at the next push. With clear, removes the layer and the module is back to what its definition says.",

internal/inventory/migrations/0055-a-token-is-issued-for-a-tunnel-key.sql

@@ -0,0 +1,7 @@
+-- A token issued for a tunnel key (novox/hq ADR 0169).
+--
+-- A machine that joins through the tunnel makes its key first, and the token is issued for it: the
+-- hub is told the key before the token is shown. So enrolment must take that key and no other — a
+-- different one is a machine the hub does not know, offering a tunnel that would never answer. Null
+-- for a token issued without one, which enrols as before.
+alter table enrolment_token add column overlay_key text;

internal/inventory/nodes.go

@@ -356,6 +356,33 @@ func (i *Inventory) Claim(ctx context.Context, secret, by string, again bool) (N
 	return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id))
 }
 
+// BindTokenToKey records the tunnel key a node's live token was issued for (novox/hq ADR 0169), so
+// enrolment takes that key and no other. Refused when the node has no live token to bind: a key
+// recorded against nothing would be a promise nothing keeps.
+func (i *Inventory) BindTokenToKey(ctx context.Context, node, key string) error {
+	tag, err := i.store.Pool().Exec(ctx,
+		`update enrolment_token set overlay_key = $2
+		  where node = $1 and redeemed is null and expires > now()`, node, key)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return fmt.Errorf("no live token to issue for the tunnel key")
+	}
+	return nil
+}
+
+// TokenKey is the tunnel key a token was issued for, or empty when it was issued without one.
+func (i *Inventory) TokenKey(ctx context.Context, secret string) (string, error) {
+	var key *string
+	err := i.store.Pool().QueryRow(ctx,
+		`select overlay_key from enrolment_token where secret = $1`, hashSecret(secret)).Scan(&key)
+	if err != nil || key == nil {
+		return "", err
+	}
+	return *key, nil
+}
+
 // Spend makes a claimed token used, only for the presenter holding the claim. The last write to the
 // store in an enrolment, so a token is spent exactly when the node it enrolled is complete. Spent
 // again by the same presenter is not an error: an answer lost after the first spend.

internal/link/bus_nats.go

@@ -12,6 +12,7 @@ func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Se
 		inbound:   Nats(js),
 		bus:       OverNATS{JS: js.Context(), Conn: js.Conn()},
 		js:        js,
+		consumers: js,
 		enroller:  enroller,
 		listener:  listener,
 		log:       newLog(),

internal/link/enrol_consumer_test.go

@@ -0,0 +1,51 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+type ensured struct{ consumers []broker.Consumer }
+
+func (e *ensured) EnsureConsumer(c broker.Consumer) error {
+	e.consumers = append(e.consumers, c)
+	return nil
+}
+
+type acceptsAs string
+
+func (n acceptsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) {
+	return EnrolReply{Accepted: true, Node: string(n)}, nil
+}
+
+type anEnrolment struct{ body []byte }
+
+func (m anEnrolment) Kind() string                         { return "enrol" }
+func (m anEnrolment) Body() []byte                         { return m.body }
+func (m anEnrolment) Redelivered() bool                    { return false }
+func (m anEnrolment) HeldFor() time.Duration               { return 0 }
+func (m anEnrolment) Answer(context.Context, []byte) error { return nil }
+func (m anEnrolment) Took() error                          { return nil }
+func (m anEnrolment) Hold(time.Duration) error             { return nil }
+func (m anEnrolment) Drop() error                          { return nil }
+
+// **A node that enrols can hear its declarations at once** (novox/hq 04-ISSUES/146): its consumer is
+// made as it enrols, not only when the control plane next starts — the first machine of a mesh
+// enrols after the control plane is up, and heard nothing.
+func TestAnEnrolledNodeIsGivenHowItHearsItsDeclarations(t *testing.T) {
+	made := &ensured{}
+	s := &Server{enroller: acceptsAs("anchor"), consumers: made, log: log.New(io.Discard, "", 0)}
+	body, _ := json.Marshal(EnrolRequest{Node: "anchor"})
+	s.enrolling(context.Background(), anEnrolment{body: body})
+
+	want := broker.NodeConsumer("anchor")
+	if len(made.consumers) != 1 || made.consumers[0].Name != want.Name || made.consumers[0].Stream != want.Stream {
+		t.Fatalf("the enrolled node was given %v, want its own declaration consumer %v", made.consumers, want)
+	}
+}

internal/link/enrol_tunnel_key_test.go

@@ -0,0 +1,37 @@
+package link_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A token issued for a tunnel key takes that key and no other** (novox/hq ADR 0169). The hub was
+// sent the key before the token was shown, so another key is a machine it does not know.
+func TestATokenIssuedForATunnelKeyTakesThatKeyAndNoOther(t *testing.T) {
+	inv, ident := aMeshReadyToEnrol(t)
+	ctx := context.Background()
+	secret, public := aTokenFor(t, inv, "joiner")
+	node, err := inv.NodeByName(ctx, "joiner")
+	if err != nil {
+		t.Fatal(err)
+	}
+	const issuedFor = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+	if err := inv.BindTokenToKey(ctx, node.ID, issuedFor); err != nil {
+		t.Fatal(err)
+	}
+	e := link.Enrolment{Inventory: inv, Identity: ident}
+
+	_, err = e.Enrol(ctx, link.EnrolRequest{Node: "joiner", Secret: secret, PublicKey: public,
+		OverlayKey: "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB="})
+	if err == nil || !strings.Contains(err.Error(), "issued for the tunnel key") {
+		t.Fatalf("a token issued for one key took another: %v", err)
+	}
+
+	if _, err := e.Enrol(ctx, link.EnrolRequest{Node: "joiner", Secret: secret, PublicKey: public,
+		OverlayKey: issuedFor}); err != nil {
+		t.Fatalf("the key the token was issued for was refused: %v", err)
+	}
+}

internal/link/enrolment.go

@@ -86,6 +86,18 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (reply Enrol
 	if err != nil {
 		return EnrolReply{}, err
 	}
+	// **A token issued for a tunnel key takes that key and no other** (novox/hq ADR 0169). The hub
+	// was told it before the token was shown; another key is a machine the hub does not know.
+	// Checked before anything is recorded, so a refusal changes nothing.
+	bound, err := e.Inventory.TokenKey(ctx, secret)
+	if err != nil {
+		return EnrolReply{}, err
+	}
+	if bound != "" && request.OverlayKey != bound {
+		return EnrolReply{}, fmt.Errorf("%s's token was issued for the tunnel key %s and the machine "+
+			"offered %q — the key it made with `nox-mesh-host key` is the one to issue for",
+			node.Name, bound, request.OverlayKey)
+	}
 
 	if _, err := e.Identity.RecordNodeKey(ctx, node.ID, public); err != nil {
 		return EnrolReply{}, fmt.Errorf("%s's key could not be recorded: %w", node.Name, err)

internal/link/serve.go

@@ -73,6 +73,8 @@ type Server struct {
 	inbound Inbound
 	bus     Bus
 	js      *broker.JetStream
+	// consumers makes a node's declaration consumer as it enrols; the bus connection, or a stand-in.
+	consumers interface{ EnsureConsumer(broker.Consumer) error }
 
 	enroller Enroller
 	listener Listener
@@ -383,6 +385,7 @@ func (s *Server) enrolling(ctx context.Context, m Control) {
 		default:
 			reply = accepted
 			s.log.Printf("enrolled %s", accepted.Node)
+			s.hearsItsDeclarations(accepted.Node)
 		}
 	}
 
@@ -402,6 +405,24 @@ func (s *Server) enrolling(ctx context.Context, m Control) {
 	_ = m.Took()
 }
 
+// hearsItsDeclarations makes the consumer a node reads its declarations through, as it enrols and
+// before it is answered.
+//
+// **Created at enrolment, as the consumer's own doc has always said** (novox/hq 04-ISSUES/146). It
+// was asserted only when the control plane started, so the first machine of a mesh — which enrols
+// after the control plane is already up — joined and then heard nothing, its host retrying "consumer
+// not found" for ever. Failing here is said and does not unspend the token: the next start of the
+// control plane asserts it again.
+func (s *Server) hearsItsDeclarations(node string) {
+	if s.consumers == nil {
+		return
+	}
+	if err := s.consumers.EnsureConsumer(broker.NodeConsumer(node)); err != nil {
+		s.log.Printf("%s enrolled, and how it hears its declarations could not be made — it will hear "+
+			"nothing until the control plane next starts: %v", node, err)
+	}
+}
+
 // wasBuilt keeps what a builder said, whichever way it went.
 //
 // This is for results nobody was waiting for. A build asked for with `build` is answered directly

internal/token/token.go

@@ -55,6 +55,26 @@ type Token struct {
 	// that it speaks the firewall found on the machine, because an adopted node keeps that firewall
 	// in force. Absent for a converged node, so a converged token is byte for byte what it was.
 	Adopted bool `json:"adopted,omitempty"`
+
+	// Tunnel is the one peer a joining machine needs, when the token was issued for its tunnel key
+	// (novox/hq ADR 0169). The machine brings its tunnel up from this alone and reaches the bus over
+	// it, at an address on the private network — so the bus is never open to the internet. Absent
+	// on a token issued without a key, which then reads byte for byte as before.
+	Tunnel *Tunnel `json:"tunnel,omitempty"`
+}
+
+// Tunnel is the joining machine's side of its first tunnel: its own address and the hub to reach.
+type Tunnel struct {
+	// Key is the public half of the key the machine made itself, which this token was issued for.
+	// The private half never left the machine (novox/hq ADR 0004).
+	Key string `json:"key"`
+	// Address is the machine's own address on the private network, with its prefix.
+	Address string `json:"address"`
+	// Range is the private network, routed through the hub until the machine is told more.
+	Range string `json:"range"`
+	// HubKey and HubEndpoint are the hub's tunnel key and where it is dialled.
+	HubKey      string `json:"hub_key"`
+	HubEndpoint string `json:"hub_endpoint"`
 }
 
 // Missing names the parts that are not filled in.
@@ -83,6 +103,19 @@ func (t Token) Missing() []string {
 	if strings.TrimSpace(t.Secret) == "" {
 		missing = append(missing, "the one-time secret — nothing to present")
 	}
+	if t.Tunnel != nil {
+		for _, part := range []struct{ value, says string }{
+			{t.Tunnel.Key, "the machine's own tunnel key — the hub would not know it"},
+			{t.Tunnel.Address, "the machine's address on the private network"},
+			{t.Tunnel.Range, "the private network's range — nothing to route through the hub"},
+			{t.Tunnel.HubKey, "the hub's tunnel key — nothing to dial"},
+			{t.Tunnel.HubEndpoint, "where the hub's tunnel is dialled"},
+		} {
+			if strings.TrimSpace(part.value) == "" {
+				missing = append(missing, part.says)
+			}
+		}
+	}
 	return missing
 }
 

internal/token/token_test.go

@@ -172,3 +172,38 @@ func TestATokenWithNoNameIsRefused(t *testing.T) {
 		t.Fatalf("the refusal does not say what is missing: %v", without.Missing())
 	}
 }
+
+// A token issued for a tunnel key carries the one peer a joining machine needs (novox/hq ADR 0169),
+// and says which part is missing rather than producing a tunnel that never answers.
+func TestATokenThroughTheTunnelCarriesThePeerOrSaysWhatIsMissing(t *testing.T) {
+	whole := Token{Node: "n", Broker: "10.42.0.1:4222", Fingerprint: "sha256:x", Signer: make([]byte, 32), Secret: "s",
+		Tunnel: &Tunnel{Key: "k", Address: "10.42.0.9/32", Range: "10.42.0.0/16", HubKey: "h", HubEndpoint: "198.51.100.1:51820"}}
+	if !whole.Complete() {
+		t.Fatalf("a whole token through the tunnel reads as missing %v", whole.Missing())
+	}
+	encoded, err := whole.Encode()
+	if err != nil {
+		t.Fatal(err)
+	}
+	back, err := Decode(encoded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if back.Tunnel == nil || *back.Tunnel != *whole.Tunnel {
+		t.Fatalf("the tunnel did not survive the round trip: %+v", back.Tunnel)
+	}
+
+	part := whole
+	part.Tunnel = &Tunnel{Key: "k", Address: "10.42.0.9/32"}
+	if len(part.Missing()) != 3 {
+		t.Errorf("a tunnel without the hub and the range should name three missing parts: %v", part.Missing())
+	}
+
+	// And a token issued without a key carries no tunnel at all, byte for byte as before.
+	plain := whole
+	plain.Tunnel = nil
+	raw, _ := plain.Encode()
+	if strings.Contains(raw, "tunnel") {
+		t.Error("a token without a key mentions a tunnel")
+	}
+}