mesh-controller/internal/link/protocol.go

// Package link is the control plane's side of the connection nodes hold open.
//
// novox/hq ADR 0002: nodes communicate over a message broker, not over HTTP. One exchange, and
// the control plane is the single consumer behind it — ADR 0006 makes that a property worth
// having rather than an accident, because two consumers sharing a queue silently split the
// traffic between them, each receiving half of what it expects. That has happened here before.
package link

import (
	"encoding/base64"
	"time"
)

// Exchange is where nodes publish everything they have to say.
const Exchange = "mesh"

// ControlQueue is what the control plane consumes. One queue, one consumer.
const ControlQueue = "control"

// Routing keys. A node may publish these; it may not publish anything else, because its broker
// account is scoped to this exchange and its own queue.
const (
	KeyEnrol  = "enrol"
	KeyReport = "report"
	KeyAlive  = "alive"
)

// QueueFor is the queue a node consumes from — the only one it may read.
func QueueFor(node string) string { return "node." + node }

// EnrolRequest is what a joining node says.
//
// It arrives on a connection the broker has already authenticated, because the account was
// created when the token was issued and the token's secret is its password. So this message is
// not how a node gets in — it is what it says once it is in.
type EnrolRequest struct {
	// Node is what this machine believes it is called. Checked against the token, never trusted.
	Node string `json:"node"`

	// Secret is the one-time right to join. The account password and this are the same string,
	// which is deliberate: the broker proves somebody holds the token, and this proves the same
	// thing to the control plane without the control plane having to ask the broker who connected.
	Secret string `json:"secret"`

	// PublicKey is what the mesh will believe from now on. The node generated it; the private
	// half has never left that machine (novox/hq ADR 0004).
	PublicKey []byte `json:"public_key"`

	// OverlayKey is the public half of this node's key on the private network — a different key
	// from PublicKey, and the mesh only ever sees this half.
	OverlayKey string `json:"overlay_key,omitempty"`

	// SealingKey is the public half of the key this node's secrets are sealed to. A third key,
	// and the reasoning is the same one twice over: the mesh must be able to send this node
	// something nothing else can read, and it must never be able to read it either.
	SealingKey string `json:"sealing_key,omitempty"`

	// ServingKey is the public half of the key this node serves TLS with on its internal name.
	// The mesh signs a certificate binding it; the private half never leaves the machine, so
	// there is nothing to seal and a copy of what the mesh holds certifies nothing new.
	ServingKey string `json:"serving_key,omitempty"`

	// Profile is what this machine can be asked to do. The control plane cannot decide what a
	// node should run without it, so it arrives with enrolment rather than being asked for after.
	Profile map[string]any `json:"profile,omitempty"`

	// Proof is the node's identity key signing EnrolProof over this request: that the presenter
	// holds the private half of PublicKey, not only knows the public one. Required to finish an
	// enrolment whose token this key already spent — the case of an answer lost after the spend —
	// because a public key is no secret, and without it anyone holding a leaked token and a
	// node's public key could replay the spent token (novox/hq issue 083, on review).
	Proof []byte `json:"proof,omitempty"`

	// Redelivered is set by the control plane, never sent: the broker handed this request over a
	// second time. Such a request does not finish an enrolment already spent — the first time may
	// have answered, and the node holds what it was told.
	Redelivered bool `json:"-"`
}

// Signed is a declaration and the signature over it.
//
// The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies
// what it received rather than what it re-encoded, because any difference in key order or spacing
// would break a signature over the same meaning.
type Signed struct {
	Declaration []byte `json:"declaration"`
	Signature   []byte `json:"signature"`
}

// Alive is a node saying nothing except that it is there.
//
// How long a node has been out of touch is a fact only the mesh can hold — nobody else is
// watching — and without it a node running last month's assignments looks exactly like one that
// is current.
type Alive struct {
	Node string `json:"node"`
}

// Report is what a node states after applying. It states; the owning context writes.
type Report struct {
	Node    string            `json:"node"`
	Applied []string          `json:"applied,omitempty"`
	Failed  map[string]string `json:"failed,omitempty"`
	Refused string            `json:"refused,omitempty"`
	// Superseded names the newer declaration the reported one was set aside for, unapplied — a
	// machine asked to be several things in a row becomes the last (novox/hq issue 031). Not an
	// account of the machine: it moves last_seen and nothing else, like a bare word that the node
	// is there, because the report for the declaration that WAS applied follows at once.
	Superseded string `json:"superseded,omitempty"`

	// Carried are the machine's ports held by what that host raised from its own bundle.
	//
	// **The half the mesh cannot know** (novox/hq ADR 0038). The foundation is not a module — a
	// node raises it before any mesh exists — so without being told, the mesh assigns a module a
	// port the store or the broker already holds, and hears about it from a container runtime.
	//
	// The node states and this context writes, which is the shape of every message here.
	Carried []int `json:"carried,omitempty"`
	// Declared is the digest of the declaration this report is about — the same bytes, hashed
	// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
	Declared string `json:"declared,omitempty"`

	// Held is what an adopted node found and is keeping as it was until its module is taken
	// (novox/hq ADR 0100). Without it an adopted node reads as converged.
	Held []Held `json:"held,omitempty"`
	// Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that
	// was never asked, which is every converged one.
	Firewall string `json:"firewall,omitempty"`
	// Reachable is what can be reached on the machine now: every listening socket and every
	// published container port. Only an adopted node reports it; it is what converging previews.
	Reachable []Reach `json:"reachable,omitempty"`
}

// Held is one file or container found on an adopted node and kept as it was.
type Held struct {
	ID     string    `json:"id"`
	Module string    `json:"module"`
	Kind   string    `json:"kind"`
	Target string    `json:"target"`
	Since  time.Time `json:"since"`
	// Changed is what something other than the mesh did to it since — rewritten, stopped,
	// replaced or gone — and empty while it is as found.
	Changed string `json:"changed,omitempty"`
	// Kept is where a file's original was kept.
	Kept string `json:"kept,omitempty"`
}

// Reach is one thing reachable on the machine: a listening socket, or a published container port.
type Reach struct {
	Protocol string `json:"protocol"`
	Address  string `json:"address"`
	Port     int    `json:"port"`
	// By is what holds it — a process, or a container's name.
	By string `json:"by,omitempty"`
	// Published is a container port the runtime publishes, reached on the forwarded path; its
	// container's own port is ContainerPort.
	Published     bool `json:"published,omitempty"`
	ContainerPort int  `json:"container-port,omitempty"`
}

// EnrolReply is what the mesh says back.
type EnrolReply struct {
	// Accepted says whether the node is now known.
	Accepted bool `json:"accepted"`

	// TryAgain says the mesh cannot answer right now — its store is restarting, or the token is
	// held for a moment by another enrolment — and the node should ask again with the same
	// request. Nothing was spent (novox/hq issue 083).
	TryAgain bool `json:"try_again,omitempty"`

	// Node is the name the mesh has for this machine, which settles any disagreement: the token
	// was issued for a node record, and that record's name wins over what the machine called
	// itself.
	Node string `json:"node,omitempty"`

	// Queue is where this node listens from now on.
	Queue string `json:"queue,omitempty"`

	// Password is this node's own broker account from now on, replacing the token's secret.
	// A credential that lives for as long as the node should not be the same string as one that
	// was meant to be used once.
	Password string `json:"password,omitempty"`

	// Fingerprint and Signer are what the node keeps so it can reconnect and keep verifying
	// without a person and a new token.
	Fingerprint string `json:"fingerprint,omitempty"`
	Signer      []byte `json:"signer,omitempty"`
	Broker      string `json:"broker,omitempty"`

	// Refusal says why not, in words for a person. Deliberately the same for every reason a
	// token can fail — unknown, spent, expired — so that guessing learns nothing.
	Refusal string `json:"refusal,omitempty"`
}

// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
// presents, so a proof cannot be moved to another request.
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
	return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
		"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
}