novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests 2 Releases Wiki Activity

Enrolment behind a seam, with both transports

Browse Source 
The last of the host's link that still named a transport. `Asking` is one
enrolment conversation — a connection made with the token, a question asked, and
an answer waited for — and it is its own seam rather than part of `Link` because
almost nothing about it is the same: the credential is a one-time secret, there
is no declaration to hear, and a node that fails here is not in the mesh at all,
where a node that fails in `Link` has merely lost touch with one it belongs to.

`Enrol`'s thirteen arguments became an `Approach` — where, which certificate,
which bus — and the request it already had. The token says nothing about which
bus, and does not need to: every token names the one the mesh runs on today until
the rollout.

**The reply address is the whole of what changes on the new bus**, and it is
forced rather than preferred. Verified against a running server, both halves: the
answer reaches the node at the address its request carried in the payload, and
the transport's own reply field held something else entirely by the time the
consumer saw it — the consumer's ack address, exactly as design 25 §2 says. The
test asserts the field is *not* the node's inbox, so a future server that stopped
claiming it would fail this rather than let the reason quietly become folklore.

The inbox is under `_INBOX.enrol.<node>.`, which is exactly what the enrolling
user may subscribe and no wider, with a random tail per attempt: a reply left
over from an attempt that timed out is not the answer to this question, which is
what the correlation id does on the other transport. Subscribed before anything
is published, because a node that published first could miss an answer to a
question nobody was listening for.
This commit is contained in:
jschoubben
2026-09-27 01:31:46 +02:00
parent 6e208f7b3e
commit 9072f60a30
7 changed files with 545 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/link/enrol.go
+49 -113
View File
@@ -6,10 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net/url"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
// The wire format shared with the control plane, which defines it separately because this binary
@@ -53,6 +50,16 @@ type EnrolRequest struct {
// on a token this key already spent (novox/hq issue 083).
Proof []byte `json:"proof,omitempty"`
// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
// transport's own reply address (design 25 §2). Written by the transport that needs it —
// withReplyTo, once per attempt — because a request going into a stream has had the transport's
// reply field claimed for the consumer's ack address before the controller ever reads it.
//
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
// field means what it has always meant. Named here so both sides of the wire hold the same
// field name, which is what the shape test on each side is for.
ReplyTo string `json:"reply_to,omitempty"`
// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
// 0105): everything about it but that key. Sent with the keys because it is one of them —
// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
@@ -147,52 +154,15 @@ func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
// was issued and the secret is its password. So this is not how the node gets in — it is what it
// says once it is in, and the secret travels again because the control plane must not have to ask
// the broker who connected.
func Enrol(ctx context.Context, address, pin, node, secret string, public []byte,
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {
config, err := PinnedConfig(pin)
if err != nil {
return EnrolReply{}, err
}
// The account name is the node's, and the password is the token's secret. Escaped because a
// name or secret containing a colon or an at-sign would otherwise change which host this
// connects to — a credential silently redirecting a connection is the worst shape this could
// take.
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
url.QueryEscape(node), url.QueryEscape(secret), address)
conn, err := amqp.DialConfig(dsn, amqp.Config{
TLSClientConfig: config,
Dial: amqp.DefaultDial(timeout),
})
if err != nil {
if errors.Is(err, ErrWrongCertificate) {
return EnrolReply{}, err
}
// Not quoted back: the DSN carries the one-time secret.
return EnrolReply{}, fmt.Errorf("cannot reach the broker at %s as %s: %w", address, node, err)
}
defer conn.Close()
channel, err := conn.Channel()
if err != nil {
return EnrolReply{}, err
}
defer channel.Close()
// This node's own queue, which its account is scoped to and nothing else may read.
queue, err := channel.QueueDeclare(QueueFor(node), true, false, false, false, nil)
if err != nil {
return EnrolReply{}, fmt.Errorf(
"cannot declare this node's queue %s: %w", QueueFor(node), err)
}
replies, err := channel.Consume(queue.Name, "", true, false, false, false, nil)
asking, err := Present(ctx, to, node, secret, timeout)
if err != nil {
return EnrolReply{}, err
}
defer asking.Close()
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
@@ -202,81 +172,47 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte
return EnrolReply{}, err
}
// Asked, and asked again with the same request while the mesh says "try again": the keys
// this node generated are the ones it keeps, so the same request is the same enrolment, and
// the mesh holds the token for it (novox/hq issue 083).
ask := func() (string, error) {
correlation := fmt.Sprintf("%s-%d", node, time.Now().UnixNano())
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
if err := channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false,
amqp.Publishing{
ContentType: "application/json",
CorrelationId: correlation,
ReplyTo: queue.Name,
Body: body,
}); err != nil {
return "", fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err)
}
return correlation, nil
}
correlation, err := ask()
if err != nil {
return EnrolReply{}, err
}
// Asked, and asked again with the same request while the mesh says "try again": the keys this
// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
// holds the token for it (novox/hq issue 083).
began := time.Now()
// Waited for rather than assumed. A published message that nothing answers means the control
// plane is not running, and a node that carried on regardless would believe it had joined a
// mesh that has never heard of it.
deadline := time.NewTimer(timeout)
defer deadline.Stop()
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
for {
answer, err := asking.Ask(ctx, body, timeout)
if err != nil {
return EnrolReply{}, err
}
var reply EnrolReply
if err := json.Unmarshal(answer, &reply); err != nil {
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
}
again, err := answered(reply, time.Since(began))
if err != nil {
return reply, err
}
if !again {
return reply, nil
}
select {
case <-ctx.Done():
return EnrolReply{}, ctx.Err()
case reason := <-closed:
return EnrolReply{}, fmt.Errorf("the broker closed the connection: %v", reason)
case <-deadline.C:
return EnrolReply{}, fmt.Errorf(
"the broker accepted this node's connection and nothing answered within %s. The "+
"mesh's broker is running and its control plane is not", timeout)
case delivery, ok := <-replies:
if !ok {
return EnrolReply{}, errors.New("the broker stopped delivering")
}
// Anything else on this queue is not the answer to this question.
if delivery.CorrelationId != correlation {
continue
}
var reply EnrolReply
if err := json.Unmarshal(delivery.Body, &reply); err != nil {
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
}
again, err := answered(reply, time.Since(began))
if err != nil {
return reply, err
}
if !again {
return reply, nil
}
select {
case <-ctx.Done():
return EnrolReply{}, ctx.Err()
case <-time.After(AskAgainAfter):
}
if correlation, err = ask(); err != nil {
return EnrolReply{}, err
}
if !deadline.Stop() {
select {
case <-deadline.C:
default:
}
}
deadline.Reset(timeout)
case <-time.After(AskAgainAfter):
}
}
}
// withReplyTo writes this attempt's reply address into the request, as a field of its own.
//
// **Written into the bytes rather than carried beside them**, because the whole point is that the
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address, so a reply address that is not in the payload is one the
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
// field before marshalling, so one request can be asked again with a fresh address each time without
// the caller knowing that is what happens.
func withReplyTo(request []byte, inbox string) ([]byte, error) {
var fields map[string]any
if err := json.Unmarshal(request, &fields); err != nil {
return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
}
fields["reply_to"] = inbox
return json.Marshal(fields)
}