mesh-controller/internal/link/protocol.go

// Package link is the control plane's side of the connection nodes hold open.
//
// novox/hq ADR 0002: nodes communicate over a message broker, not over HTTP. One exchange, and
// the control plane is the single consumer behind it — ADR 0006 makes that a property worth
// having rather than an accident, because two consumers sharing a queue silently split the
// traffic between them, each receiving half of what it expects. That has happened here before.
package link

import (
	"encoding/base64"
	"strconv"
	"strings"
	"time"
)

// Exchange is where nodes publish everything they have to say.
const Exchange = "mesh"

// ControlQueue is what the control plane consumes. One queue, one consumer.
const ControlQueue = "control"

// Routing keys. A node may publish these; it may not publish anything else, because its broker
// account is scoped to this exchange and its own queue.
const (
	KeyEnrol  = "enrol"
	KeyReport = "report"
	KeyAlive  = "alive"
)

// QueueFor is the queue a node consumes from — the only one it may read.
func QueueFor(node string) string { return "node." + node }

// EnrolRequest is what a joining node says.
//
// It arrives on a connection the broker has already authenticated, because the account was
// created when the token was issued and the token's secret is its password. So this message is
// not how a node gets in — it is what it says once it is in.
type EnrolRequest struct {
	// Node is what this machine believes it is called. Checked against the token, never trusted.
	Node string `json:"node"`

	// Secret is the one-time right to join. The account password and this are the same string,
	// which is deliberate: the broker proves somebody holds the token, and this proves the same
	// thing to the control plane without the control plane having to ask the broker who connected.
	Secret string `json:"secret"`

	// PublicKey is what the mesh will believe from now on. The node generated it; the private
	// half has never left that machine (novox/hq ADR 0004).
	PublicKey []byte `json:"public_key"`

	// OverlayKey is the public half of this node's key on the private network — a different key
	// from PublicKey, and the mesh only ever sees this half.
	OverlayKey string `json:"overlay_key,omitempty"`

	// SealingKey is the public half of the key this node's secrets are sealed to. A third key,
	// and the reasoning is the same one twice over: the mesh must be able to send this node
	// something nothing else can read, and it must never be able to read it either.
	SealingKey string `json:"sealing_key,omitempty"`

	// ServingKey is the public half of the key this node serves TLS with on its internal name.
	// The mesh signs a certificate binding it; the private half never leaves the machine, so
	// there is nothing to seal and a copy of what the mesh holds certifies nothing new.
	ServingKey string `json:"serving_key,omitempty"`

	// Profile is what this machine can be asked to do. The control plane cannot decide what a
	// node should run without it, so it arrives with enrolment rather than being asked for after.
	Profile map[string]any `json:"profile,omitempty"`

	// Proof is the node's identity key signing EnrolProof over this request: that the presenter
	// holds the private half of PublicKey, not only knows the public one. Required to finish an
	// enrolment whose token this key already spent — the case of an answer lost after the spend —
	// because a public key is no secret, and without it anyone holding a leaked token and a
	// node's public key could replay the spent token (novox/hq issue 083, on review).
	Proof []byte `json:"proof,omitempty"`

	// Tunnel is the tunnel this node found on its machine and whose key it took as its overlay
	// key (novox/hq ADR 0105): the interface, its port, address and range, and its peers. Presented
	// with the keys because it is one of them — OverlayKey above is this tunnel's public key when
	// it is set — and the mesh composes the hub's address, the range and every carried peer from
	// it. Nil from a node that found none, which is every converged one.
	Tunnel *Tunnel `json:"tunnel,omitempty"`

	// ReplyTo is where the answer goes, as a field of the request rather than the transport's own
	// reply address.
	//
	// **Because a stream eats the transport's field** (design 25 §2, verified against a running
	// server): a message a JetStream consumer delivers has had its reply field claimed for that
	// consumer's own ack address, so by the time the controller sees an enrolment, the field names
	// where the *controller* must acknowledge, not where the node is waiting. An enrolment is the
	// case that matters — a caller waiting on an ephemeral inbox, over a subject the store window
	// may legitimately delay by several nak cycles.
	//
	// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
	// field means what it has always meant.
	ReplyTo string `json:"reply_to,omitempty"`

	// Redelivered is set by the control plane, never sent: the broker handed this request over a
	// second time. Such a request does not finish an enrolment already spent — the first time may
	// have answered, and the node holds what it was told.
	Redelivered bool `json:"-"`
}

// Tunnel is a found tunnel as a node presents it: everything but its private key, which the node
// keeps as its own overlay key and never sends.
type Tunnel struct {
	Interface string       `json:"interface"`
	Unit      string       `json:"unit"`
	Config    string       `json:"config"`
	Port      int          `json:"port"`
	MTU       int          `json:"mtu,omitempty"`
	Address   string       `json:"address"`
	Range     string       `json:"range"`
	PublicKey string       `json:"public_key"`
	Peers     []TunnelPeer `json:"peers,omitempty"`
}

// TunnelPeer is one peer of a found tunnel: its public key and the address the tunnel routed to
// it.
type TunnelPeer struct {
	PublicKey string `json:"public_key"`
	Address   string `json:"address"`
}

// Signed is a declaration and the signature over it.
//
// The signature is over Declaration exactly as it will arrive, bytes unchanged — a node verifies
// what it received rather than what it re-encoded, because any difference in key order or spacing
// would break a signature over the same meaning.
type Signed struct {
	Declaration []byte `json:"declaration"`
	Signature   []byte `json:"signature"`
}

// Alive is a node saying nothing except that it is there.
//
// How long a node has been out of touch is a fact only the mesh can hold — nobody else is
// watching — and without it a node running last month's assignments looks exactly like one that
// is current.
type Alive struct {
	Node string `json:"node"`
	// IntervalSeconds is how often the node says it is there (novox/hq to-be 45 §3, S1): the
	// watchdog's bound is three of them. Zero from a host older than that, which is read as the
	// interval hosts have always used.
	IntervalSeconds int `json:"interval_seconds,omitempty"`
}

// ToolsAlive is a machine's node tools saying they are there (novox/hq to-be 45 §3, S11): the
// runtime every module's tools and every held seat's verbs are served by. Its own word, apart from the
// host's, because a host heard and a runtime gone is a machine nobody can ask anything.
type ToolsAlive struct {
	Node            string `json:"node"`
	IntervalSeconds int    `json:"interval_seconds,omitempty"`
	// Version is the runtime's build, as it says it.
	Version string `json:"version,omitempty"`
}

// Report is what a node states after applying. It states; the owning context writes.
type Report struct {
	Node    string            `json:"node"`
	Applied []string          `json:"applied,omitempty"`
	Failed  map[string]string `json:"failed,omitempty"`
	Refused string            `json:"refused,omitempty"`
	// Superseded names the newer declaration the reported one was set aside for, unapplied — a
	// machine asked to be several things in a row becomes the last (novox/hq issue 031). Not an
	// account of the machine: it moves last_seen and nothing else, like a bare word that the node
	// is there, because the report for the declaration that WAS applied follows at once.
	Superseded string `json:"superseded,omitempty"`

	// Carried are the machine's ports held by what that host raised from its own bundle.
	//
	// **The half the mesh cannot know** (novox/hq ADR 0038). The foundation is not a module — a
	// node raises it before any mesh exists — so without being told, the mesh assigns a module a
	// port the store or the broker already holds, and hears about it from a container runtime.
	//
	// The node states and this context writes, which is the shape of every message here.
	Carried []int `json:"carried,omitempty"`
	// Declared is the digest of the declaration this report is about — the same bytes, hashed
	// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
	Declared string `json:"declared,omitempty"`

	// Order is where the declaration this report is about stands — its `epoch` and `sequence` as the
	// declaration carried them — so the mesh keeps accounts by what they are about rather than by when
	// they arrived (novox/hq to-be 45 §6; the contract is order.go). Two top-level keys; absent for a
	// declaration that claimed no order, and from a node-engine older than the contract.
	Order
	// ReportSequence is the node-engine's own number for this report: one higher for every report it
	// makes, kept on disk across restarts and self-updates. Zero claims none — every report an older
	// node-engine makes. Its presence also says the node-engine reads a declaration's epoch.
	ReportSequence int64 `json:"report_sequence,omitempty"`
	// OlderThan is set on a report refusing a declaration older than one the machine applied: the order
	// of the one it holds. The refused declaration is the report's own Declared and Order — whose epoch
	// names the controller that sent it (S13).
	OlderThan *Order `json:"older_than,omitempty"`
	// RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report:
	// a refusal whose own report was lost is still counted from the next.
	RefusedOlder int64 `json:"refused_older,omitempty"`

	// Held is what an adopted node found and is keeping as it was until its module is taken
	// (novox/hq ADR 0100). Without it an adopted node reads as converged.
	Held []Held `json:"held,omitempty"`
	// Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that
	// was never asked, which is every converged one.
	Firewall string `json:"firewall,omitempty"`

	// Outward is the links on this machine that face outside it — the ones carrying a default route
	// (novox/hq ADR 0140). Every node reports it, adopted or converged, because the filter the mesh
	// composes for it is written around these and nothing else.
	//
	// **It replaces a list of addresses.** The filter used to block everything passing through the
	// machine and then allow the machine's own containers back by naming the ranges they sit on. A
	// range describes one machine and goes stale in silence; the link carrying the default route is
	// read afresh on every report and does not change when a module is added or removed.
	//
	// Empty means the machine has not said. The mesh composes no filter for such a machine and
	// leaves the one it has: a rule written around a link with no name is a rule set that does not
	// load, and that is a machine filtering nothing while its unit reports success.
	Outward []string `json:"outward,omitempty"`

	// Host is the version of the host that produced this report (novox/hq ADR 0141).
	//
	// **The machine has sent this since 0141 and this struct did not have it**, so it was
	// unmarshalled into nothing and the mesh could not say which host any machine runs
	// (novox/hq 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and
	// refuses it whole — which is right, and makes every new field a flag day that the mesh could
	// not see coming.
	Host string `json:"host,omitempty"`

	// Strays is what runs on the machine that the mesh neither wrote nor holds (ADR 0163).
	Strays []Stray `json:"strays,omitempty"`

	// Filters is what filters the machine now: every table and chain that refuses traffic, with
	// its owner — the mesh's, the found firewall's, the container runtime's own, a ban, or other
	// (novox/hq ADR 0168). Every machine reports it, adopted or converged; absent from a host older
	// than this.
	Filters []Filter `json:"filters,omitempty"`
	// FoundFirewall is the state of the firewall a converged machine was found with: in force now
	// or not, and how it came to be inactive — the mesh disabled it, or it was found so (ADR 0168).
	FoundFirewall *FoundFirewall `json:"found_firewall,omitempty"`

	// Profile is what the machine can do, detected again by this apply (novox/hq ADR 0161): the
	// same shape enrolment sends, so a machine that gained or lost a capability — switched its
	// network manager — is known at its next push and not at its next enrolment. Absent from a host
	// older than this, and then the enrolment's profile stands.
	Profile map[string]any `json:"profile,omitempty"`
	// Reachable is what can be reached on the machine now: every listening socket and every
	// published container port. Only an adopted node reports it; it is what converging previews.
	Reachable []Reach `json:"reachable,omitempty"`

	// Tunnel is what an adopted node says about the tunnel it found and carried (novox/hq ADR
	// 0105): the interface, its port, range and peer count, whether the found interface is down
	// and the mesh's up in its place, and where the found configuration's original was kept.
	Tunnel *CarriedTunnel `json:"tunnel,omitempty"`

	// Rekey is a node taking a found tunnel's key as its overlay key after enrolment (novox/hq
	// ADR 0105). A report carrying one is not an account of the machine: it moves the node's
	// overlay key and tunnel and nothing else.
	Rekey *Rekey `json:"rekey,omitempty"`
}

// CarriedTunnel is a node's account of the tunnel it took over. State is "not-taken" (the found
// interface still up, the mesh's not), "taken" (the found one down, the mesh's up with its key) or
// "down" (the found one down and the mesh's not up: the peers reach nothing); Note is what the host
// did about it.
type CarriedTunnel struct {
	Interface string `json:"interface"`
	Port      int    `json:"port"`
	Range     string `json:"range"`
	Peers     int    `json:"peers"`
	State     string `json:"state"`
	Note      string `json:"note,omitempty"`
	Kept      string `json:"kept,omitempty"`
}

// Rekey is a node saying it took a found tunnel's key as its overlay key after enrolling (novox/hq
// ADR 0105) — the path for a hub that enrolled before the mesh knew to take a tunnel over, since
// re-enrolling would rotate every key the node holds. Carried in a report, on the node's own
// authenticated connection, and signed with its identity key over RekeyProof, so a report forged
// on a stolen broker account cannot move a node's overlay key.
type Rekey struct {
	// Previous is the overlay key the node holds now, as the mesh records it. A rekey naming
	// another is stale — a replay, or made against a record that moved on — and is refused.
	Previous   string  `json:"previous"`
	OverlayKey string  `json:"overlay_key"`
	Tunnel     *Tunnel `json:"tunnel"`
	Proof      []byte  `json:"proof"`
}

// RekeyProof is what a node signs when it rekeys: the node, the key it leaves, the key it takes
// and the tunnel it took it from, so a proof cannot be moved to another node or another tunnel.
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
	var t Tunnel
	if tunnel != nil {
		t = *tunnel
	}
	peers := make([]string, 0, len(t.Peers))
	for _, p := range t.Peers {
		peers = append(peers, p.PublicKey+"@"+p.Address)
	}
	return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
		t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
		t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
}

// Held is one file or container found on an adopted node and kept as it was.
type Held struct {
	ID     string    `json:"id"`
	Module string    `json:"module"`
	Kind   string    `json:"kind"`
	Target string    `json:"target"`
	Since  time.Time `json:"since"`
	// Changed is what something other than the mesh did to it since — rewritten, stopped,
	// replaced or gone — and empty while it is as found.
	Changed string `json:"changed,omitempty"`
	// Kept is where a file's original was kept.
	Kept string `json:"kept,omitempty"`
	// Facts is the found thing beside what the module declares — what a take compares (novox/hq
	// ADR 0163): the host's own shape, carried as data and read by the preview.
	Facts map[string]any `json:"facts,omitempty"`
}

// A Filter is one place on a machine that refuses traffic, with its owner (novox/hq ADR 0168):
// the host's own shape, carried as data.
type Filter struct {
	Where   string `json:"where"`
	Owner   string `json:"owner"`
	Refuses string `json:"refuses"`
}

// FoundFirewall is the state of a converged machine's found firewall (ADR 0168).
type FoundFirewall struct {
	Kind      string `json:"kind"`
	Active    bool   `json:"active"`
	RetiredBy string `json:"retired_by,omitempty"`
}

// A Stray is a container a machine runs that the mesh neither wrote nor holds (ADR 0163).
type Stray struct {
	Kind   string `json:"kind"`
	Name   string `json:"name"`
	Detail string `json:"detail,omitempty"`
}

// Reach is one thing reachable on the machine: a listening socket, or a published container port.
type Reach struct {
	Protocol string `json:"protocol"`
	Address  string `json:"address"`
	Port     int    `json:"port"`
	// By is what holds it — a process, or a container's name.
	By string `json:"by,omitempty"`
	// Published is a container port the runtime publishes, reached on the forwarded path; its
	// container's own port is ContainerPort.
	Published     bool `json:"published,omitempty"`
	ContainerPort int  `json:"container-port,omitempty"`
}

// EnrolReply is what the mesh says back.
type EnrolReply struct {
	// Accepted says whether the node is now known.
	Accepted bool `json:"accepted"`

	// TryAgain says the mesh cannot answer right now — its store is restarting, or the token is
	// held for a moment by another enrolment — and the node should ask again with the same
	// request. Nothing was spent (novox/hq issue 083).
	TryAgain bool `json:"try_again,omitempty"`

	// Node is the name the mesh has for this machine, which settles any disagreement: the token
	// was issued for a node record, and that record's name wins over what the machine called
	// itself.
	Node string `json:"node,omitempty"`

	// Queue is where this node listens from now on.
	Queue string `json:"queue,omitempty"`

	// Password is this node's own broker account from now on, replacing the token's secret.
	// A credential that lives for as long as the node should not be the same string as one that
	// was meant to be used once.
	Password string `json:"password,omitempty"`

	// Fingerprint and Signer are what the node keeps so it can reconnect and keep verifying
	// without a person and a new token.
	Fingerprint string `json:"fingerprint,omitempty"`
	Signer      []byte `json:"signer,omitempty"`
	Broker      string `json:"broker,omitempty"`

	// Refusal says why not, in words for a person. Deliberately the same for every reason a
	// token can fail — unknown, spent, expired — so that guessing learns nothing.
	Refusal string `json:"refusal,omitempty"`
}

// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
// presents, so a proof cannot be moved to another request.
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
	return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
		"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
}