mesh-controller/internal/broker/nats.go

// Composing the bus's own configuration.
//
// An account is *composed*, never called for: the controller writes accounts, users and
// per-subject permissions into one file the host keeps current, and the server reloads it in
// place (novox/hq ADR 0106 — never through a management API; design 25 §4).
//
// Everything here is pure. Given the principals, it returns the file's text — so the whole of the
// mesh's authority model is testable as strings, with no server.
//
// **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042
// reserves a module's origin — it publishes only under its own name — and here that is a refusal
// rather than something a library promises.
package broker

import (
	"errors"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

// A Kind is what a principal is, which decides the shape of its authority rather than its
// contents: a module's comes from its declaration, a host's from its node, and the controller's
// and the enrolment user's are fixed.
type Kind string

const (
	KindModule     Kind = "module"
	KindNode       Kind = "node"
	KindController Kind = "controller"
	KindEnrolment  Kind = "enrolment"
	// KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its
	// authority is a list of tools and nothing else — not control, not declarations, not builds,
	// and no ability to answer anything, because a person asks.
	KindPerson Kind = "person"
)

// Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it
// emits (novox/hq ADR 0118, design 29 §5).
type Seat struct {
	Name     string
	Accepts  []string
	Emits    []string
	Serves   []string
	Versions []string // protocol versions served beside the current one; empty for v1 only
}

// A Principal is one user of the bus. Its permissions are derived from what it declares and
// nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats
// it holds, and the seats it uses.
type Principal struct {
	Kind   Kind
	Node   string
	Module string

	Emits    []string
	Consumes []string
	Serves   []string

	Holds []Seat
	Uses  []Seat

	// Invokes are the tools a person may call, as `<module>.<tool>`; a single `*` is every tool,
	// for an administrator. Only meaningful for KindPerson.
	//
	// **A list, not a role.** A person is not a module and holds no seat: nothing is addressed
	// to them, nothing is delivered to them, and they have no durable consumer to acknowledge.
	// What they have is permission to ask.
	Invokes []string

	// PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal
	// and never appears here: this file is written to a node's disk and read by a server, and a
	// secret that can be read from a configuration file is a secret with a wider blast radius
	// than the one it protects (novox/hq design 29 §10).
	PasswordHash string
}

// enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the
// controller may answer an enrolment is derived from the same constant the user is named from.
const enrolmentPrefix = "enrol"

// safeSubject refuses anything that would change the meaning of a subject rather than sit inside
// one. A name carrying a dot would silently widen a permission by adding a token; a name carrying
// `>` or `*` would widen it to a wildcard, which is the whole authority model gone.
var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)

// Username is how a principal is named to the server. The node is part of it, so the same module
// on two machines holds two users, each sealed to its own — the rule management.go already
// applies, kept.
func (p Principal) Username() string {
	switch p.Kind {
	case KindPerson:
		return "person." + p.Module
	case KindModule:
		return p.Node + "." + p.Module
	case KindNode:
		return "node." + p.Node
	case KindController:
		return "controller"
	case KindEnrolment:
		// Per token, not one shared user. **The inbox is the reason**: with a single `enrolment`
		// user every machine enrolling at once could read every other's answer, and an answer
		// carries that node's credentials sealed to it. Design 25 §6 says the inbox a token
		// derives, and a permission belongs to a user, so the user is per token.
		//
		// Named after the node, which **is** the token's id: a token is issued for a node record,
		// the mesh holds one live claim per record, and the node's name is the one identifier both
		// sides already have before anything else is agreed. It is also exactly what the other
		// transport does, where the account is named after the node and the secret is its password.
		return enrolmentPrefix + "." + p.Node
	}
	return ""
}

// inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25
// §4): with one account, inbox privacy is the permission list or it is nothing, so each user's
// inbox is derived from its own identity and its permissions name that prefix and no other.
func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" }

// Permissions is what a principal may publish and subscribe, and whether it may answer.
type Permissions struct {
	Publish   []string
	Subscribe []string
	// AllowResponses lets a principal reply to a request it received, on the reply subject that
	// request carried, once.
	//
	// **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If
	// every user's inbox is private to it, a module serving a tool cannot publish the answer —
	// the answer goes to the *caller's* inbox, which the responder has no permission for. The two
	// ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4
	// refuses, or this: the server itself permits one reply to the subject of a message the user
	// actually received, and nothing else. The authority is bounded by having been asked.
	AllowResponses bool
}

// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
// a permission that cannot be derived from a declaration is a permission nobody can explain.
func PermissionsFor(p Principal) (Permissions, error) {
	for _, part := range []struct{ what, value string }{
		{"node", p.Node}, {"module", p.Module},
	} {
		if part.value == "" {
			continue
		}
		if !safeSubject.MatchString(part.value) {
			return Permissions{}, fmt.Errorf(
				"%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value)
		}
	}

	var pub, sub []string
	switch p.Kind {
	case KindController:
		// The controller owns the mesh's own traffic and the streams. It is the only writer of
		// stream definitions (design 25 §3), so it alone reaches the JetStream API.
		pub = []string{"mesh.control.>", "mesh.node.>", "mesh.build.>", "$JS.API.>"}
		sub = []string{"mesh.control.>", "mesh.build.>", "$JS.API.>"}

		// The two events it reacts to, and its ack subject on the stream they arrive from
		// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
		// controller a subscriber to every event in the mesh, and its permission list would stop
		// saying what it is for. The ack grant below is scoped per stream because the controller's
		// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
		// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
		// ever, refused by the list it already has.
		sub = append(sub, ControllerFollows...)
		pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>")

		// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
		// permits one reply to the reply subject of a message the user received — and a message a
		// JetStream consumer delivers has had that field claimed for the consumer's own ack address
		// (design 25 §2), so the address the controller actually answers is the one the request
		// carried in its payload, which is not a reply subject as the server understands it.
		//
		// Verified against a real server before this line existed: the answer was refused with
		// "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the
		// mesh would have timed out while the controller logged success.
		//
		// **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and
		// this is not it: nothing but an enrolling node ever subscribes under this prefix, each
		// scoped to its own token's, so the controller publishing here is the mesh answering
		// enrolments and can reach nothing else.
		pub = append(pub, "_INBOX."+enrolmentPrefix+".>")

	case KindPerson:
		// Tools, and nothing else. Every subject a person may publish is a tool call; a person
		// who could publish an event would be able to claim a module said something.
		for _, t := range p.Invokes {
			if t == "*" {
				pub = append(pub, "mesh.mod.*.tool.>")
				continue
			}
			module, tool, ok := strings.Cut(t, ".")
			if !ok {
				return Permissions{}, fmt.Errorf(
					"%q does not name a tool: a person invokes <module>.<tool>, or * for every one", t)
			}
			pub = append(pub, "mesh.mod."+module+".tool."+tool)
		}

	case KindEnrolment:
		// A leaked token is useless for anything but enrolling: it cannot read a declaration, hear
		// an event, or subscribe any inbox but the one its own token derives (design 25 §6).
		//
		// **The inbox was missing and the handshake could not have completed without it.** An
		// enrolling node publishes its request and waits on an address it states in the payload;
		// with nothing to subscribe it waits out its timeout against a mesh that answered. Its own
		// and no wider: `_INBOX.enrol.<node>.>`, so what is sealed to one machine cannot be read by
		// another enrolling beside it.
		if p.Node == "" {
			// Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty
			// token in it — and worse, one every nameless enrolment user would share. A shared
			// enrolment inbox is one machine able to read the credentials sealed to another.
			return Permissions{}, errors.New(
				"an enrolment user names no node, so its inbox would be shared with every other " +
					"enrolment: a token is issued for a node record, and that record's name is " +
					"the token's id")
		}
		pub = []string{"mesh.control.enrol"}
		sub = []string{p.inbox()}

	case KindNode:
		// A host publishes its own node's control traffic and subscribes its own declaration —
		// and nothing of any other node's.
		pub = []string{"mesh.control." + p.Node + ".>"}
		sub = []string{"mesh.node." + p.Node + ".declare"}

	case KindModule:
		// 1. Its own namespace: it publishes its events there and serves its tools there. Nothing
		//    else may publish into it, so an event's source is a fact the server enforces rather
		//    than a claim in the body (design 29 §2).
		own := "mesh.mod." + p.Module
		for _, e := range p.Emits {
			pub = append(pub, own+".event."+e)
		}
		for _, t := range p.Serves {
			sub = append(sub, own+".tool."+t)
		}

		// 2. What it consumes, by the emitter's own subject — an event is addressed to its
		//    emitter, because the emitter's identity is the meaning (ADR 0118).
		for _, c := range p.Consumes {
			emitter, event, ok := strings.Cut(c, ".")
			if !ok {
				return Permissions{}, fmt.Errorf(
					"%q does not name an emitter and an event: a consumed event is <module>.<event>", c)
			}
			sub = append(sub, "mesh.mod."+emitter+".event."+event)
		}

		// 3. Seats it holds: full participation.
		for _, s := range p.Holds {
			for _, a := range s.Accepts {
				sub = append(sub, seatSubject(s, "accept", a))
			}
			for _, e := range s.Emits {
				pub = append(pub, seatSubject(s, "event", e))
			}
			for _, t := range s.Serves {
				sub = append(sub, seatSubject(s, "tool", t))
			}
		}

		// 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a
		//    seat's inbound subject and watch other modules' traffic, nor publish its outbound
		//    events and lie about outcomes (design 29 §2).
		for _, s := range p.Uses {
			for _, a := range s.Accepts {
				pub = append(pub, seatSubject(s, "accept", a))
			}
			for _, t := range s.Serves {
				pub = append(pub, seatSubject(s, "tool", t))
			}
		}
	}

	if p.Kind == KindPerson {
		// An inbox to hear answers in, and nothing else. No ack subject: a person has no durable
		// consumer, because nothing is delivered to a person — they ask and are answered.
		sub = append(sub, p.inbox())
	}

	if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController {
		// Its own reply space, and nothing wider.
		sub = append(sub, p.inbox())

		// Acking a JetStream delivery is a publish to that consumer's own ack address — a
		// different subject from anything the consumer subscribes. Without it every message a
		// module received would be redelivered forever, refused by the permission list it already
		// has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own
		// deliveries and no other's.
		pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>")
	}

	sort.Strings(pub)
	sort.Strings(sub)
	return Permissions{
		Publish:   pub,
		Subscribe: sub,
		// Only something that serves is ever answering. A pure consumer is granted nothing here.
		AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) ||
			p.Kind == KindController,
	}, nil
}

// seatSubject places a seat's verb under the kind of traffic it is.
//
// **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so
// without it a stream over a seat or a module's namespace would capture that namespace's *tool*
// traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS).
// Found while defining the streams: the first draft of design 29 had one namespace per module
// with no kind, which reads well and cannot be filtered.
//
// A seat serving more than its current protocol version carries the version as a token
// (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is
// bound to the old one.
func seatSubject(s Seat, kind, verb string) string {
	return "mesh.seat." + s.Name + "." + kind + "." + verb
}

// consumerStream and consumerDurable are the two halves of a consumer's identity, and they are
// two functions because conflating them was a real bug.
//
// **A durable name may not contain a dot; an ack subject is built from two names that do.** The
// server acknowledges on `$JS.ACK.<stream>.<consumer>.…`, so a single string "EVENTS.one_audit"
// reads correctly inside the permission and is rejected as a consumer name — *nats: invalid
// consumer name*. Caught against a running server, and worth the comment because the shape of
// the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack
// has every message redelivered forever, and its permission list looks right while it happens.
//
// They are derived here, beside the permission that must match them, because two places deriving
// the same name is how a module ends up unable to ack its own deliveries.
func consumerStream(p Principal) string {
	switch p.Kind {
	case KindModule:
		return "EVENTS"
	case KindNode:
		return "NODES"
	case KindController:
		return "CONTROL"
	}
	return ""
}

func consumerDurable(p Principal) string {
	switch p.Kind {
	case KindModule:
		return p.Node + "_" + p.Module
	case KindNode:
		return p.Node
	case KindController:
		return "controller"
	}
	return ""
}

// Server is everything the composed file needs that is not a principal.
type Server struct {
	// ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable
	// without TLS is one a module can reach without TLS by mistake.
	ClientPort     int
	MonitoringPort int
	TLSCert        string
	TLSKey         string
	TLSCA          string
	// StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted
	// four modules away from named volumes; the bus's own data is not the place to bring one back.
	StoreDir string
}

// Compose renders the server's whole configuration. The order is stable and the output is
// deterministic, because the file's digest is what the module's entrypoint watches to decide
// whether to reload: a composer that reordered a map on each run would signal a reload every time
// the controller restarted, for a file that had not changed.
func Compose(s Server, principals []Principal) (string, error) {
	sorted := append([]Principal(nil), principals...)
	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })

	var b strings.Builder
	b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n")
	b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n")
	b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n")

	fmt.Fprintf(&b, "port: %d\n", s.ClientPort)
	fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort)

	// **No `verify`, and it said `verify: true` until this configuration was run.** That setting
	// makes the server demand a *client* certificate, and nothing in the mesh presents one: a host
	// pins this server's exact certificate and authenticates with the password the mesh minted
	// (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the
	// mesh is refused at the TLS handshake before any password is looked at, and the error —
	// "client didn't provide a certificate" — reads as a fault in the client.
	//
	// TLS is still required: the block is what requires it, and verify only decides whether client
	// certificates are checked.
	b.WriteString("tls {\n")
	fmt.Fprintf(&b, "  cert_file: %q\n", s.TLSCert)
	fmt.Fprintf(&b, "  key_file: %q\n", s.TLSKey)
	fmt.Fprintf(&b, "  ca_file: %q\n", s.TLSCA)
	b.WriteString("}\n\n")

	b.WriteString("jetstream {\n")
	fmt.Fprintf(&b, "  store_dir: %q\n", s.StoreDir)
	b.WriteString("}\n\n")

	accounts, err := ComposeAccounts(sorted)
	if err != nil {
		return "", err
	}
	b.WriteString(accounts)
	return b.String(), nil
}

// ComposeAccounts is the accounts block alone — every user, and nothing about the server.
//
// **This is the only part of the configuration the mesh writes, and the split is deliberate.** A
// server's ports, its TLS paths and its store directory are properties of the container the module
// raises: they live in its image and its mounts, and they change when it does. The controller has no
// business knowing them, and a controller that did would have to be kept in step with a Dockerfile
// it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the
// module's own configuration includes it.
//
// Four things checked against a running server before this shape was committed to: a user in an
// included file authenticates; an unknown user is refused, so the include is the whole authority
// rather than an addition to something; a publish outside a user's grant is refused; and rewriting
// this file alone and signalling a reload makes a new user appear **without dropping the connection
// the mesh already has** — which is what makes every later account, permission or person change cost
// nothing (task 1.2's payoff).
func ComposeAccounts(principals []Principal) (string, error) {
	sorted := append([]Principal(nil), principals...)
	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() })

	var b strings.Builder
	b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n")
	b.WriteString("# composition overwrites it. Permissions are derived from what each module\n")
	b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n")

	// One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is
	// one space (design 25 §4). The cost of that — that permissions are the only isolation — is
	// paid in the scoping of every inbox and every ack subject.
	b.WriteString("accounts {\n  MESH {\n    users = [\n")
	for _, p := range sorted {
		perms, err := PermissionsFor(p)
		if err != nil {
			return "", err
		}
		if p.PasswordHash == "" {
			return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username())
		}
		fmt.Fprintf(&b, "      { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash)
		fmt.Fprintf(&b, "        publish: { allow: [%s] }\n", quoted(perms.Publish))
		fmt.Fprintf(&b, "        subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
		if perms.AllowResponses {
			b.WriteString("        allow_responses: { max: 1, ttl: \"1m\" }\n")
		}
		b.WriteString("      } }\n")
	}
	b.WriteString("    ]\n  }\n}\n")
	return b.String(), nil
}

func quoted(values []string) string {
	if len(values) == 0 {
		return ""
	}
	out := make([]string, len(values))
	for i, v := range values {
		out[i] = fmt.Sprintf("%q", v)
	}
	return strings.Join(out, ", ")
}