// Package link is the control plane's side of the connection nodes hold open. // // novox/hq ADR 0002: nodes communicate over a message broker, not over HTTP. One exchange, and // the control plane is the single consumer behind it — ADR 0006 makes that a property worth // having rather than an accident, because two consumers sharing a queue silently split the // traffic between them, each receiving half of what it expects. That has happened here before. package link import ( "encoding/base64" "strconv" "strings" "time" ) // Exchange is where nodes publish everything they have to say. const Exchange = "mesh" // ControlQueue is what the control plane consumes. One queue, one consumer. const ControlQueue = "control" // Routing keys. A node may publish these; it may not publish anything else, because its broker // account is scoped to this exchange and its own queue. const ( KeyEnrol = "enrol" KeyReport = "report" KeyAlive = "alive" ) // QueueFor is the queue a node consumes from — the only one it may read. func QueueFor(node string) string { return "node." + node } // EnrolRequest is what a joining node says. // // It arrives on a connection the broker has already authenticated, because the account was // created when the token was issued and the token's secret is its password. So this message is // not how a node gets in — it is what it says once it is in. type EnrolRequest struct { // Node is what this machine believes it is called. Checked against the token, never trusted. Node string `json:"node"` // Secret is the one-time right to join. The account password and this are the same string, // which is deliberate: the broker proves somebody holds the token, and this proves the same // thing to the control plane without the control plane having to ask the broker who connected. Secret string `json:"secret"` // PublicKey is what the mesh will believe from now on. The node generated it; the private // half has never left that machine (novox/hq ADR 0004). PublicKey []byte `json:"public_key"` // OverlayKey is the public half of this node's key on the private network — a different key // from PublicKey, and the mesh only ever sees this half. OverlayKey string `json:"overlay_key,omitempty"` // SealingKey is the public half of the key this node's secrets are sealed to. A third key, // and the reasoning is the same one twice over: the mesh must be able to send this node // something nothing else can read, and it must never be able to read it either. SealingKey string `json:"sealing_key,omitempty"` // ServingKey is the public half of the key this node serves TLS with on its internal name. // The mesh signs a certificate binding it; the private half never leaves the machine, so // there is nothing to seal and a copy of what the mesh holds certifies nothing new. ServingKey string `json:"serving_key,omitempty"` // Profile is what this machine can be asked to do. The control plane cannot decide what a // node should run without it, so it arrives with enrolment rather than being asked for after. Profile map[string]any `json:"profile,omitempty"` // Proof is the node's identity key signing EnrolProof over this request: that the presenter // holds the private half of PublicKey, not only knows the public one. Required to finish an // enrolment whose token this key already spent — the case of an answer lost after the spend — // because a public key is no secret, and without it anyone holding a leaked token and a // node's public key could replay the spent token (novox/hq issue 083, on review). Proof []byte `json:"proof,omitempty"` // Tunnel is the tunnel this node found on its machine and whose key it took as its overlay // key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented // with the keys because it is one of them — OverlayKey above is this tunnel's public key when // it is set — and the mesh composes the hub's address, the range and every carried peer from // it. Nil from a node that found none, which is every converged one. Tunnel *Tunnel `json:"tunnel,omitempty"` // Redelivered is set by the control plane, never sent: the broker handed this request over a // second time. Such a request does not finish an enrolment already spent — the first time may // have answered, and the node holds what it was told. Redelivered bool `json:"-"` } // Tunnel is a found tunnel as a node presents it: everything but its private key, which the node // keeps as its own overlay key and never sends. type Tunnel struct { Interface string `json:"interface"` Unit string `json:"unit"` Config string `json:"config"` Port int `json:"port"` MTU int `json:"mtu,omitempty"` Address string `json:"address"` Range string `json:"range"` PublicKey string `json:"public_key"` Peers []TunnelPeer `json:"peers,omitempty"` } // TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to // it. type TunnelPeer struct { PublicKey string `json:"public_key"` Address string `json:"address"` } // Signed is a declaration and the signature over it. // // The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies // what it received rather than what it re-encoded, because any difference in key order or spacing // would break a signature over the same meaning. type Signed struct { Declaration []byte `json:"declaration"` Signature []byte `json:"signature"` } // Alive is a node saying nothing except that it is there. // // How long a node has been out of touch is a fact only the mesh can hold — nobody else is // watching — and without it a node running last month's assignments looks exactly like one that // is current. type Alive struct { Node string `json:"node"` } // Report is what a node states after applying. It states; the owning context writes. type Report struct { Node string `json:"node"` Applied []string `json:"applied,omitempty"` Failed map[string]string `json:"failed,omitempty"` Refused string `json:"refused,omitempty"` // Superseded names the newer declaration the reported one was set aside for, unapplied — a // machine asked to be several things in a row becomes the last (novox/hq issue 031). Not an // account of the machine: it moves last_seen and nothing else, like a bare word that the node // is there, because the report for the declaration that WAS applied follows at once. Superseded string `json:"superseded,omitempty"` // Carried are the machine's ports held by what that host raised from its own bundle. // // **The half the mesh cannot know** (novox/hq ADR 0038). The foundation is not a module — a // node raises it before any mesh exists — so without being told, the mesh assigns a module a // port the store or the broker already holds, and hears about it from a container runtime. // // The node states and this context writes, which is the shape of every message here. Carried []int `json:"carried,omitempty"` // Declared is the digest of the declaration this report is about — the same bytes, hashed // the same way, as the `sent` digest the mesh recorded. Which declaration, not when. Declared string `json:"declared,omitempty"` // Held is what an adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` // Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that // was never asked, which is every converged one. Firewall string `json:"firewall,omitempty"` // Reachable is what can be reached on the machine now: every listening socket and every // published container port. Only an adopted node reports it; it is what converging previews. Reachable []Reach `json:"reachable,omitempty"` // Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR // 0105): the interface, its port, range and peer count, whether the found interface is down // and the mesh's up in its place, and where the found configuration's original was kept. Tunnel *CarriedTunnel `json:"tunnel,omitempty"` // Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq // ADR 0105). A report carrying one is not an account of the machine: it moves the node's // overlay key and tunnel and nothing else. Rekey *Rekey `json:"rekey,omitempty"` } // CarriedTunnel is a node's account of the tunnel it took over. State is "not-taken" (the found // interface still up, the mesh's not), "taken" (the found one down, the mesh's up with its key) or // "down" (the found one down and the mesh's not up: the peers reach nothing); Note is what the host // did about it. type CarriedTunnel struct { Interface string `json:"interface"` Port int `json:"port"` Range string `json:"range"` Peers int `json:"peers"` State string `json:"state"` Note string `json:"note,omitempty"` Kept string `json:"kept,omitempty"` } // Rekey is a node saying it took a found tunnel's key as its overlay key after enrolling (novox/hq // ADR 0105) — the path for a hub that enrolled before the mesh knew to take a tunnel over, since // re-enrolling would rotate every key the node holds. Carried in a report, on the node's own // authenticated connection, and signed with its identity key over RekeyProof, so a report forged // on a stolen broker account cannot move a node's overlay key. type Rekey struct { // Previous is the overlay key the node holds now, as the mesh records it. A rekey naming // another is stale — a replay, or made against a record that moved on — and is refused. Previous string `json:"previous"` OverlayKey string `json:"overlay_key"` Tunnel *Tunnel `json:"tunnel"` Proof []byte `json:"proof"` } // RekeyProof is what a node signs when it rekeys: the node, the key it leaves, the key it takes // and the tunnel it took it from, so a proof cannot be moved to another node or another tunnel. func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { var t Tunnel if tunnel != nil { t = *tunnel } peers := make([]string, 0, len(t.Peers)) for _, p := range t.Peers { peers = append(peers, p.PublicKey+"@"+p.Address) } return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) } // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` Module string `json:"module"` Kind string `json:"kind"` Target string `json:"target"` Since time.Time `json:"since"` // Changed is what something other than the mesh did to it since — rewritten, stopped, // replaced or gone — and empty while it is as found. Changed string `json:"changed,omitempty"` // Kept is where a file's original was kept. Kept string `json:"kept,omitempty"` } // Reach is one thing reachable on the machine: a listening socket, or a published container port. type Reach struct { Protocol string `json:"protocol"` Address string `json:"address"` Port int `json:"port"` // By is what holds it — a process, or a container's name. By string `json:"by,omitempty"` // Published is a container port the runtime publishes, reached on the forwarded path; its // container's own port is ContainerPort. Published bool `json:"published,omitempty"` ContainerPort int `json:"container-port,omitempty"` } // EnrolReply is what the mesh says back. type EnrolReply struct { // Accepted says whether the node is now known. Accepted bool `json:"accepted"` // TryAgain says the mesh cannot answer right now — its store is restarting, or the token is // held for a moment by another enrolment — and the node should ask again with the same // request. Nothing was spent (novox/hq issue 083). TryAgain bool `json:"try_again,omitempty"` // Node is the name the mesh has for this machine, which settles any disagreement: the token // was issued for a node record, and that record's name wins over what the machine called // itself. Node string `json:"node,omitempty"` // Queue is where this node listens from now on. Queue string `json:"queue,omitempty"` // Password is this node's own broker account from now on, replacing the token's secret. // A credential that lives for as long as the node should not be the same string as one that // was meant to be used once. Password string `json:"password,omitempty"` // Fingerprint and Signer are what the node keeps so it can reconnect and keep verifying // without a person and a new token. Fingerprint string `json:"fingerprint,omitempty"` Signer []byte `json:"signer,omitempty"` Broker string `json:"broker,omitempty"` // Refusal says why not, in words for a person. Deliberately the same for every reason a // token can fail — unknown, spent, expired — so that guessing learns nothing. Refusal string `json:"refusal,omitempty"` } // EnrolProof is what a node signs with its identity key when it enrols: the token and every key it // presents, so a proof cannot be moved to another request. func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + "\x00" + overlay + "\x00" + sealing + "\x00" + serving) }