mesh-host/internal/link/enrol.go

package link

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"time"
)

// The wire format shared with the control plane, which defines it separately because this binary
// requires nothing present and does not import it. A test on each side asserts the field names.
const (
	Exchange = "mesh"
	KeyEnrol = "enrol"
)

// QueueFor is the queue this node consumes from — the only one its account may read.
func QueueFor(node string) string { return "node." + node }

// EnrolRequest is what this node says when joining.
type EnrolRequest struct {
	Node      string `json:"node"`
	Secret    string `json:"secret"`
	PublicKey []byte `json:"public_key"`

	// OverlayKey is the public half of this node's key on the private network — a different key
	// from PublicKey above, generated at the same moment and for a different purpose.
	//
	// Sent with enrolment because the overlay is the first declaration a node receives, and the
	// mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled
	// and unreachable for a round trip, which is the state everything else here works to avoid.
	OverlayKey string `json:"overlay_key,omitempty"`

	// SealingKey is the public half of the key secrets are sealed to. A third key, and the
	// reasoning is the same one twice over: the mesh must be able to send this node something
	// nothing else can read, and it must never be able to read it either.
	SealingKey string `json:"sealing_key,omitempty"`

	// ServingKey is the public half of the key this node serves TLS with on its internal name.
	// The mesh signs a certificate binding it; the private half never leaves the machine, so
	// there is nothing to seal and nothing that could be stolen from the mesh's copy.
	ServingKey string `json:"serving_key,omitempty"`

	Profile map[string]any `json:"profile,omitempty"`

	// Proof is this node's identity key signing EnrolProof over this request: that the presenter
	// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
	// on a token this key already spent (novox/hq issue 083).
	Proof []byte `json:"proof,omitempty"`

	// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
	// transport's own reply address (design 25 §2). Written by the transport that needs it —
	// withReplyTo, once per attempt — because a request going into a stream has had the transport's
	// reply field claimed for the consumer's ack address before the controller ever reads it.
	//
	// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
	// field means what it has always meant. Named here so both sides of the wire hold the same
	// field name, which is what the shape test on each side is for.
	ReplyTo string `json:"reply_to,omitempty"`

	// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
	// 0105): everything about it but that key. Sent with the keys because it is one of them —
	// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
	// hub's address, the range and the carried peers from it before the first declaration.
	Tunnel *Tunnel `json:"tunnel,omitempty"`
}

// Tunnel is a found tunnel as it travels: no private key.
type Tunnel struct {
	Interface string       `json:"interface"`
	Unit      string       `json:"unit"`
	Config    string       `json:"config"`
	Port      int          `json:"port"`
	Address   string       `json:"address"`
	Range     string       `json:"range"`
	PublicKey string       `json:"public_key"`
	Peers     []TunnelPeer `json:"peers,omitempty"`
}

// TunnelPeer is one peer of a found tunnel: its key, and the address the tunnel routes to it.
type TunnelPeer struct {
	PublicKey string `json:"public_key"`
	Address   string `json:"address"`
}

// EnrolReply is what the mesh says back.
type EnrolReply struct {
	Accepted bool `json:"accepted"`

	// TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the
	// token is held for a moment by another enrolment — and that nothing was spent. The same
	// request is asked again (novox/hq issue 083).
	TryAgain bool `json:"try_again,omitempty"`

	Node  string `json:"node,omitempty"`
	Queue string `json:"queue,omitempty"`

	// What this node keeps so it can come back on its own. Without these a restart would need a
	// person with a new token, which would make disconnection a crisis rather than the ordinary
	// situation novox/hq ADR 0004 says it is.
	Password    string `json:"password,omitempty"`
	Broker      string `json:"broker,omitempty"`
	Fingerprint string `json:"fingerprint,omitempty"`
	Signer      []byte `json:"signer,omitempty"`

	Refusal string `json:"refusal,omitempty"`
}

// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
// presents, so a proof cannot be moved to another request. The mesh builds the same bytes.
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
	return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
		"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
}

// ErrRefused is what a node gets when the mesh will not have it.
var ErrRefused = errors.New("the mesh refused this enrolment")

// EnrolPatience is how long a node keeps asking while the mesh says "try again" — as long as the
// mesh holds a token for the one enrolment presenting it, so a node asking the whole time is never
// held off by its own earlier attempt.
const EnrolPatience = 2 * time.Minute

// AskAgainAfter is the pause between asks while the mesh says "try again".
const AskAgainAfter = 3 * time.Second

// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking.
var ErrNotNow = errors.New("the mesh could not answer this enrolment")

// answered decides what one reply means: done, ask again, or stop with an error. Separate from the
// broker so it can be held to that by a test.
func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
	switch {
	case reply.Accepted:
		return false, nil
	case reply.TryAgain && asking < EnrolPatience:
		return true, nil
	case reply.TryAgain:
		// Said with what to do. The mesh holds the token for this attempt's keys for as long as
		// this node kept asking, so a new attempt — with keys of its own — waits that out first.
		return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+
			"enrol again with it; if it is then refused, issue a new one",
			ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience)
	default:
		return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal)
	}
}

// Enrol presents this node's key and its one-time secret, and waits to be told it is known.
//
// The broker has already authenticated this connection: the account was created when the token
// was issued and the secret is its password. So this is not how the node gets in — it is what it
// says once it is in, and the secret travels again because the control plane must not have to ask
// the broker who connected.
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
	overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
	tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {

	asking, err := Present(ctx, to, node, secret, timeout)
	if err != nil {
		return EnrolReply{}, err
	}
	defer asking.Close()

	request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
		OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
		Proof: proof, Tunnel: tunnel}
	body, err := json.Marshal(request)
	if err != nil {
		return EnrolReply{}, err
	}

	// Asked, and asked again with the same request while the mesh says "try again": the keys this
	// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
	// holds the token for it (novox/hq issue 083).
	began := time.Now()
	for {
		answer, err := asking.Ask(ctx, body, timeout)
		if err != nil {
			return EnrolReply{}, err
		}
		var reply EnrolReply
		if err := json.Unmarshal(answer, &reply); err != nil {
			return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
		}
		again, err := answered(reply, time.Since(began))
		if err != nil {
			return reply, err
		}
		if !again {
			return reply, nil
		}
		select {
		case <-ctx.Done():
			return EnrolReply{}, ctx.Err()
		case <-time.After(AskAgainAfter):
		}
	}
}

// withReplyTo writes this attempt's reply address into the request, as a field of its own.
//
// **Written into the bytes rather than carried beside them**, because the whole point is that the
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address, so a reply address that is not in the payload is one the
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
// field before marshalling, so one request can be asked again with a fresh address each time without
// the caller knowing that is what happens.
func withReplyTo(request []byte, inbox string) ([]byte, error) {
	var fields map[string]any
	if err := json.Unmarshal(request, &fields); err != nil {
		return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
	}
	fields["reply_to"] = inbox
	return json.Marshal(fields)
}