package link import ( "context" "encoding/base64" "encoding/json" "errors" "fmt" "time" ) // The wire format shared with the control plane, which defines it separately because this binary // requires nothing present and does not import it. A test on each side asserts the field names. const ( Exchange = "mesh" KeyEnrol = "enrol" ) // QueueFor is the queue this node consumes from — the only one its account may read. func QueueFor(node string) string { return "node." + node } // EnrolRequest is what this node says when joining. type EnrolRequest struct { Node string `json:"node"` Secret string `json:"secret"` PublicKey []byte `json:"public_key"` // OverlayKey is the public half of this node's key on the private network — a different key // from PublicKey above, generated at the same moment and for a different purpose. // // Sent with enrolment because the overlay is the first declaration a node receives, and the // mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled // and unreachable for a round trip, which is the state everything else here works to avoid. OverlayKey string `json:"overlay_key,omitempty"` // SealingKey is the public half of the key secrets are sealed to. A third key, and the // reasoning is the same one twice over: the mesh must be able to send this node something // nothing else can read, and it must never be able to read it either. SealingKey string `json:"sealing_key,omitempty"` // ServingKey is the public half of the key this node serves TLS with on its internal name. // The mesh signs a certificate binding it; the private half never leaves the machine, so // there is nothing to seal and nothing that could be stolen from the mesh's copy. ServingKey string `json:"serving_key,omitempty"` Profile map[string]any `json:"profile,omitempty"` // Proof is this node's identity key signing EnrolProof over this request: that the presenter // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish // on a token this key already spent (novox/hq issue 083). Proof []byte `json:"proof,omitempty"` // ReplyTo is where the mesh's answer goes, as a field of the request rather than the // transport's own reply address (design 25 §2). Written by the transport that needs it — // withReplyTo, once per attempt — because a request going into a stream has had the transport's // reply field claimed for the consumer's ack address before the controller ever reads it. // // Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the // field means what it has always meant. Named here so both sides of the wire hold the same // field name, which is what the shape test on each side is for. ReplyTo string `json:"reply_to,omitempty"` // Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR // 0105): everything about it but that key. Sent with the keys because it is one of them — // OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the // hub's address, the range and the carried peers from it before the first declaration. Tunnel *Tunnel `json:"tunnel,omitempty"` } // Tunnel is a found tunnel as it travels: no private key. type Tunnel struct { Interface string `json:"interface"` Unit string `json:"unit"` Config string `json:"config"` Port int `json:"port"` Address string `json:"address"` Range string `json:"range"` PublicKey string `json:"public_key"` Peers []TunnelPeer `json:"peers,omitempty"` } // TunnelPeer is one peer of a found tunnel: its key, and the address the tunnel routes to it. type TunnelPeer struct { PublicKey string `json:"public_key"` Address string `json:"address"` } // EnrolReply is what the mesh says back. type EnrolReply struct { Accepted bool `json:"accepted"` // TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the // token is held for a moment by another enrolment — and that nothing was spent. The same // request is asked again (novox/hq issue 083). TryAgain bool `json:"try_again,omitempty"` Node string `json:"node,omitempty"` Queue string `json:"queue,omitempty"` // What this node keeps so it can come back on its own. Without these a restart would need a // person with a new token, which would make disconnection a crisis rather than the ordinary // situation novox/hq ADR 0004 says it is. Password string `json:"password,omitempty"` Broker string `json:"broker,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` Signer []byte `json:"signer,omitempty"` Refusal string `json:"refusal,omitempty"` } // EnrolProof is what a node signs with its identity key when it enrols: the token and every key it // presents, so a proof cannot be moved to another request. The mesh builds the same bytes. func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + "\x00" + overlay + "\x00" + sealing + "\x00" + serving) } // ErrRefused is what a node gets when the mesh will not have it. var ErrRefused = errors.New("the mesh refused this enrolment") // EnrolPatience is how long a node keeps asking while the mesh says "try again" — as long as the // mesh holds a token for the one enrolment presenting it, so a node asking the whole time is never // held off by its own earlier attempt. const EnrolPatience = 2 * time.Minute // AskAgainAfter is the pause between asks while the mesh says "try again". const AskAgainAfter = 3 * time.Second // ErrNotNow is a mesh that said "try again" for longer than this node would keep asking. var ErrNotNow = errors.New("the mesh could not answer this enrolment") // answered decides what one reply means: done, ask again, or stop with an error. Separate from the // broker so it can be held to that by a test. func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { switch { case reply.Accepted: return false, nil case reply.TryAgain && asking < EnrolPatience: return true, nil case reply.TryAgain: // Said with what to do. The mesh holds the token for this attempt's keys for as long as // this node kept asking, so a new attempt — with keys of its own — waits that out first. return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+ "enrol again with it; if it is then refused, issue a new one", ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience) default: return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) } } // Enrol presents this node's key and its one-time secret, and waits to be told it is known. // // The broker has already authenticated this connection: the account was created when the token // was issued and the secret is its password. So this is not how the node gets in — it is what it // says once it is in, and the secret travels again because the control plane must not have to ask // the broker who connected. func Enrol(ctx context.Context, to Approach, node, secret string, public []byte, overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) { asking, err := Present(ctx, to, node, secret, timeout) if err != nil { return EnrolReply{}, err } defer asking.Close() request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, Proof: proof, Tunnel: tunnel} body, err := json.Marshal(request) if err != nil { return EnrolReply{}, err } // Asked, and asked again with the same request while the mesh says "try again": the keys this // node generated are the ones it keeps, so the same request is the same enrolment, and the mesh // holds the token for it (novox/hq issue 083). began := time.Now() for { answer, err := asking.Ask(ctx, body, timeout) if err != nil { return EnrolReply{}, err } var reply EnrolReply if err := json.Unmarshal(answer, &reply); err != nil { return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err) } again, err := answered(reply, time.Since(began)) if err != nil { return reply, err } if !again { return reply, nil } select { case <-ctx.Done(): return EnrolReply{}, ctx.Err() case <-time.After(AskAgainAfter): } } } // withReplyTo writes this attempt's reply address into the request, as a field of its own. // // **Written into the bytes rather than carried beside them**, because the whole point is that the // address survives a stream: a JetStream consumer's delivery has had the transport's reply field // claimed for its own ack address, so a reply address that is not in the payload is one the // controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the // field before marshalling, so one request can be asked again with a fresh address each time without // the caller knowing that is what happens. func withReplyTo(request []byte, inbox string) ([]byte, error) { var fields map[string]any if err := json.Unmarshal(request, &fields); err != nil { return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err) } fields["reply_to"] = inbox return json.Marshal(fields) }