// Composing the bus's own configuration. // // An account is *composed*, never called for: the controller writes accounts, users and // per-subject permissions into one file the host keeps current, and the server reloads it in // place (novox/hq ADR 0106 — never through a management API; design 25 §4). // // Everything here is pure. Given the principals, it returns the file's text — so the whole of the // mesh's authority model is testable as strings, with no server. // // **Permissions are per subject, so a module's own name is the server's to enforce.** ADR 0042 // reserves a module's origin — it publishes only under its own name — and here that is a refusal // rather than something a library promises. package broker import ( "errors" "fmt" "regexp" "sort" "strings" ) // A Kind is what a principal is, which decides the shape of its authority rather than its // contents: a module's comes from its declaration, a host's from its node, and the controller's // and the enrolment user's are fixed. type Kind string const ( KindModule Kind = "module" KindNode Kind = "node" KindController Kind = "controller" KindEnrolment Kind = "enrolment" // KindPerson is somebody reaching the mesh's tools from a workstation (design 25 §7). Its // authority is a list of tools and nothing else — not control, not declarations, not builds, // and no ability to answer anything, because a person asks. KindPerson Kind = "person" ) // Seat is a role on the bus as a principal relates to it: the subjects it accepts, and those it // emits (novox/hq ADR 0118, design 29 §5). type Seat struct { Name string Accepts []string Emits []string Serves []string Versions []string // protocol versions served beside the current one; empty for v1 only } // A Principal is one user of the bus. Its permissions are derived from what it declares and // nothing else (novox/hq ADR 0043), over the three namespaces of design 29 §2: its own, the seats // it holds, and the seats it uses. type Principal struct { Kind Kind Node string Module string Emits []string Consumes []string Serves []string Holds []Seat Uses []Seat // Watches are seats whose events this principal consumes. Separate from Consumes because a // role's event lives under the seat's namespace and not a module's, and this package cannot tell // a seat's name from a module's by looking at it — whoever resolved the declaration can, and // does (novox/hq ADR 0121). // // **Found by a consumer reading nothing.** The catalogue consumes the build machine's outcome; // with that name read as a module's, its subscription pointed at `mesh.mod.mesh-build-machine.…`, // a namespace no such module owns. Every service started and the graph stayed empty. Watches []Seat // Invokes are the tools a person may call, as `.`; a single `*` is every tool, // for an administrator. Only meaningful for KindPerson. // // **A list, not a role.** A person is not a module and holds no seat: nothing is addressed // to them, nothing is delivered to them, and they have no durable consumer to acknowledge. // What they have is permission to ask. Invokes []string // PasswordHash is the bcrypt hash the mesh minted. The plaintext is sealed to the principal // and never appears here: this file is written to a node's disk and read by a server, and a // secret that can be read from a configuration file is a secret with a wider blast radius // than the one it protects (novox/hq design 29 §10). PasswordHash string } // meshSeatsTheControllerUses are the roles the mesh's own flows submit work to. Named rather than // derived from the seat set: the controller is not a module and declares no `uses`, so its side of a // seat has to be stated, and a list is what makes "which roles does the mesh itself talk to" answerable. var meshSeatsTheControllerUses = []string{"mesh-build-machine"} // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the // controller may answer an enrolment is derived from the same constant the user is named from. const enrolmentPrefix = "enrol" // safeSubject refuses anything that would change the meaning of a subject rather than sit inside // one. A name carrying a dot would silently widen a permission by adding a token; a name carrying // `>` or `*` would widen it to a wildcard, which is the whole authority model gone. var safeSubject = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) // Username is how a principal is named to the server. The node is part of it, so the same module // on two machines holds two users, each sealed to its own — the rule management.go already // applies, kept. func (p Principal) Username() string { switch p.Kind { case KindPerson: return "person." + p.Module case KindModule: return p.Node + "." + p.Module case KindNode: return "node." + p.Node case KindController: return "controller" case KindEnrolment: // Per token, not one shared user. **The inbox is the reason**: with a single `enrolment` // user every machine enrolling at once could read every other's answer, and an answer // carries that node's credentials sealed to it. Design 25 §6 says the inbox a token // derives, and a permission belongs to a user, so the user is per token. // // Named after the node, which **is** the token's id: a token is issued for a node record, // the mesh holds one live claim per record, and the node's name is the one identifier both // sides already have before anything else is agreed. It is also exactly what the other // transport does, where the account is named after the node and the secret is its password. return enrolmentPrefix + "." + p.Node } return "" } // inbox is a principal's own reply space. No user is ever granted a bare `_INBOX.>` (design 25 // §4): with one account, inbox privacy is the permission list or it is nothing, so each user's // inbox is derived from its own identity and its permissions name that prefix and no other. func (p Principal) inbox() string { return "_INBOX." + p.Username() + ".>" } // Permissions is what a principal may publish and subscribe, and whether it may answer. type Permissions struct { Publish []string Subscribe []string // AllowResponses lets a principal reply to a request it received, on the reply subject that // request carried, once. // // **This is what makes scoped inboxes possible at all**, and design 25 §4 did not say it. If // every user's inbox is private to it, a module serving a tool cannot publish the answer — // the answer goes to the *caller's* inbox, which the responder has no permission for. The two // ways out are granting responders `_INBOX.>`, which is precisely the blanket grant §4 // refuses, or this: the server itself permits one reply to the subject of a message the user // actually received, and nothing else. The authority is bounded by having been asked. AllowResponses bool } // PermissionsFor derives a principal's authority. Pure, and the only place authority is decided: // a permission that cannot be derived from a declaration is a permission nobody can explain. func PermissionsFor(p Principal) (Permissions, error) { for _, part := range []struct{ what, value string }{ {"node", p.Node}, {"module", p.Module}, } { if part.value == "" { continue } if !safeSubject.MatchString(part.value) { return Permissions{}, fmt.Errorf( "%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", part.value) } } var pub, sub []string switch p.Kind { case KindController: // The controller owns the mesh's own traffic and the streams. It is the only writer of // stream definitions (design 25 §3), so it alone reaches the JetStream API. pub = []string{"mesh.control.>", "mesh.node.>", "$JS.API.>"} sub = []string{"mesh.control.>", "$JS.API.>"} // Work the mesh's own flows submit to a role, and the outcomes they wait on (ADR 0121). A // build is the one today: the controller asks, and reads the answer from the seat's event // like the catalogue does — which is why no holder needs to publish into anybody's inbox. for _, seat := range meshSeatsTheControllerUses { pub = append(pub, "mesh.seat."+seat+".accept.>") } // The two events it reacts to, and its ack subject on the stream they arrive from // (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the // controller a subscriber to every event in the mesh, and its permission list would stop // saying what it is for. The ack grant below is scoped per stream because the controller's // consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a // delivery from EVENTS — a consumer that cannot ack has every message redelivered for // ever, refused by the list it already has. sub = append(sub, ControllerFollows...) pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>") // **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That // permits one reply to the reply subject of a message the user received — and a message a // JetStream consumer delivers has had that field claimed for the consumer's own ack address // (design 25 §2), so the address the controller actually answers is the one the request // carried in its payload, which is not a reply subject as the server understands it. // // Verified against a real server before this line existed: the answer was refused with // "Permissions Violation for Publish to _INBOX.enrol.anchor…", and every enrolment on the // mesh would have timed out while the controller logged success. // // **The enrolment inbox space, not a blanket `_INBOX.>`.** Design 25 §4 refuses that, and // this is not it: nothing but an enrolling node ever subscribes under this prefix, each // scoped to its own token's, so the controller publishing here is the mesh answering // enrolments and can reach nothing else. pub = append(pub, "_INBOX."+enrolmentPrefix+".>") case KindPerson: // Tools, and nothing else. Every subject a person may publish is a tool call; a person // who could publish an event would be able to claim a module said something. for _, t := range p.Invokes { if t == "*" { pub = append(pub, "mesh.mod.*.tool.>") continue } module, tool, ok := strings.Cut(t, ".") if !ok { return Permissions{}, fmt.Errorf( "%q does not name a tool: a person invokes ., or * for every one", t) } pub = append(pub, "mesh.mod."+module+".tool."+tool) } case KindEnrolment: // A leaked token is useless for anything but enrolling: it cannot read a declaration, hear // an event, or subscribe any inbox but the one its own token derives (design 25 §6). // // **The inbox was missing and the handshake could not have completed without it.** An // enrolling node publishes its request and waits on an address it states in the payload; // with nothing to subscribe it waits out its timeout against a mesh that answered. Its own // and no wider: `_INBOX.enrol..>`, so what is sealed to one machine cannot be read by // another enrolling beside it. if p.Node == "" { // Refused rather than composed into `_INBOX.enrol..>`, which is a subject with an empty // token in it — and worse, one every nameless enrolment user would share. A shared // enrolment inbox is one machine able to read the credentials sealed to another. return Permissions{}, errors.New( "an enrolment user names no node, so its inbox would be shared with every other " + "enrolment: a token is issued for a node record, and that record's name is " + "the token's id") } pub = []string{"mesh.control.enrol"} sub = []string{p.inbox()} case KindNode: // A host publishes its own node's control traffic and subscribes its own declaration — // and nothing of any other node's. pub = []string{"mesh.control." + p.Node + ".>"} sub = []string{"mesh.node." + p.Node + ".declare"} case KindModule: // 1. Its own namespace: it publishes its events there and serves its tools there. Nothing // else may publish into it, so an event's source is a fact the server enforces rather // than a claim in the body (design 29 §2). own := "mesh.mod." + p.Module for _, e := range p.Emits { pub = append(pub, own+".event."+e) } for _, t := range p.Serves { sub = append(sub, own+".tool."+t) } // 2. What it consumes, by the emitter's own subject — an event is addressed to its // emitter, because the emitter's identity is the meaning (ADR 0118). for _, c := range p.Consumes { subject, err := consumedSubject(c) if err != nil { return Permissions{}, err } sub = append(sub, subject) } // 2b. Events of a role it watches, under the seat's own namespace. Subscribe only: watching a // role is hearing what it announced, not taking part in it. for _, w := range p.Watches { for _, e := range w.Emits { sub = append(sub, seatSubject(w, "event", e)) } } // 3. Seats it holds: full participation. for _, s := range p.Holds { for _, a := range s.Accepts { sub = append(sub, seatSubject(s, "accept", a)) } for _, e := range s.Emits { pub = append(pub, seatSubject(s, "event", e)) } for _, t := range s.Serves { sub = append(sub, seatSubject(s, "tool", t)) } } // 4. Seats it uses: publish only, and only the accepts half. A caller cannot subscribe a // seat's inbound subject and watch other modules' traffic, nor publish its outbound // events and lie about outcomes (design 29 §2). for _, s := range p.Uses { for _, a := range s.Accepts { pub = append(pub, seatSubject(s, "accept", a)) } for _, t := range s.Serves { pub = append(pub, seatSubject(s, "tool", t)) } } } if p.Kind == KindPerson { // An inbox to hear answers in, and nothing else. No ack subject: a person has no durable // consumer, because nothing is delivered to a person — they ask and are answered. sub = append(sub, p.inbox()) } if p.Kind == KindModule || p.Kind == KindNode || p.Kind == KindController { // Its own reply space, and nothing wider. sub = append(sub, p.inbox()) // Acking a JetStream delivery is a publish to that consumer's own ack address — a // different subject from anything the consumer subscribes. Without it every message a // module received would be redelivered forever, refused by the permission list it already // has (design 25 §4). Scoped to this principal's own consumer name, so it can ack its own // deliveries and no other's. pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>") } sort.Strings(pub) sort.Strings(sub) return Permissions{ Publish: pub, Subscribe: sub, // Only something that serves is ever answering. A pure consumer is granted nothing here. AllowResponses: p.Kind == KindModule && (len(p.Serves) > 0 || len(p.Holds) > 0) || p.Kind == KindController, }, nil } // seatSubject places a seat's verb under the kind of traffic it is. // // **The kind token is load-bearing, not decoration.** A stream is defined by a subject filter, so // without it a stream over a seat or a module's namespace would capture that namespace's *tool* // traffic too — and a tool call must never be persisted (design 25 §3: tools stay on core NATS). // Found while defining the streams: the first draft of design 29 had one namespace per module // with no kind, which reads well and cannot be filtered. // // A seat serving more than its current protocol version carries the version as a token // (design 29 §8): the seat stays one role, and v1 and v2 run beside each other until nothing is // bound to the old one. func seatSubject(s Seat, kind, verb string) string { return "mesh.seat." + s.Name + "." + kind + "." + verb } // consumerStream and consumerDurable are the two halves of a consumer's identity, and they are // two functions because conflating them was a real bug. // // **A durable name may not contain a dot; an ack subject is built from two names that do.** The // server acknowledges on `$JS.ACK...…`, so a single string "EVENTS.one_audit" // reads correctly inside the permission and is rejected as a consumer name — *nats: invalid // consumer name*. Caught against a running server, and worth the comment because the shape of // the failure if it had not been is the one design 25 §4 warns about: a consumer that cannot ack // has every message redelivered forever, and its permission list looks right while it happens. // // They are derived here, beside the permission that must match them, because two places deriving // the same name is how a module ends up unable to ack its own deliveries. // consumedSubject is where a consumed event lands, from the local pattern a module declared. // // **The mesh's wildcards become this transport's** (design 29 §1): `*` is one name on both, and `**` // — the rest — is `>` here. A module writes neither transport's spelling, so a manifest stays correct // when the wire changes, which is the whole reason names are local. // // `**` on its own is every event from every module: the emitter is any, the event is anything. An // audit logger wants exactly that and says so in one token. func consumedSubject(pattern string) (string, error) { if pattern == catalogueTheRest { return "mesh.mod.*.event.>", nil } emitter, event, named := strings.Cut(pattern, ".") if !named || emitter == "" || event == "" { return "", fmt.Errorf( "%q does not name an emitter and an event: a consumed event is ., or "+ "%q for every event", pattern, catalogueTheRest) } if emitter == catalogueTheRest { return "", fmt.Errorf("%q stands for the rest of a name, so it cannot name the emitter", catalogueTheRest) } // Each name is checked before it becomes a subject: a name carrying a dot would add a token and // silently widen the permission, which is the whole reason safeSubject exists. var out []string for _, part := range strings.Split(event, ".") { switch part { case catalogueTheRest: out = append(out, ">") case "*": out = append(out, "*") default: if !safeSubject.MatchString(part) { return "", fmt.Errorf("%q cannot be part of a subject: it would widen the permission", part) } out = append(out, part) } } if emitter != "*" && !safeSubject.MatchString(emitter) { return "", fmt.Errorf("%q cannot name an emitter: it would widen the permission", emitter) } return "mesh.mod." + emitter + ".event." + strings.Join(out, "."), nil } // catalogueTheRest is the mesh's wildcard for "the rest of a name", duplicated from the catalogue // package for the one direction of dependency the build queue's name is duplicated for. const catalogueTheRest = "**" func consumerStream(p Principal) string { switch p.Kind { case KindModule: return "EVENTS" case KindNode: return "NODES" case KindController: return "CONTROL" } return "" } func consumerDurable(p Principal) string { switch p.Kind { case KindModule: return p.Node + "_" + p.Module case KindNode: return p.Node case KindController: return "controller" } return "" } // Server is everything the composed file needs that is not a principal. type Server struct { // ClientPort carries TLS itself. There is no plaintext port beside it: a bus reachable // without TLS is one a module can reach without TLS by mistake. ClientPort int MonitoringPort int TLSCert string TLSKey string TLSCA string // StoreDir is a host directory bind, not a named volume — issue 115 is resolved and converted // four modules away from named volumes; the bus's own data is not the place to bring one back. StoreDir string } // Compose renders the server's whole configuration. The order is stable and the output is // deterministic, because the file's digest is what the module's entrypoint watches to decide // whether to reload: a composer that reordered a map on each run would signal a reload every time // the controller restarted, for a file that had not changed. func Compose(s Server, principals []Principal) (string, error) { sorted := append([]Principal(nil), principals...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) var b strings.Builder b.WriteString("# Composed by the mesh controller. Do not edit: the next composition overwrites it.\n") b.WriteString("# Accounts and permissions are derived from what each module declares and nothing\n") b.WriteString("# else (novox/hq ADR 0043, design 29 §2).\n\n") fmt.Fprintf(&b, "port: %d\n", s.ClientPort) fmt.Fprintf(&b, "http: 127.0.0.1:%d\n\n", s.MonitoringPort) // **No `verify`, and it said `verify: true` until this configuration was run.** That setting // makes the server demand a *client* certificate, and nothing in the mesh presents one: a host // pins this server's exact certificate and authenticates with the password the mesh minted // (ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection in the // mesh is refused at the TLS handshake before any password is looked at, and the error — // "client didn't provide a certificate" — reads as a fault in the client. // // TLS is still required: the block is what requires it, and verify only decides whether client // certificates are checked. b.WriteString("tls {\n") fmt.Fprintf(&b, " cert_file: %q\n", s.TLSCert) fmt.Fprintf(&b, " key_file: %q\n", s.TLSKey) fmt.Fprintf(&b, " ca_file: %q\n", s.TLSCA) b.WriteString("}\n\n") b.WriteString("jetstream {\n") fmt.Fprintf(&b, " store_dir: %q\n", s.StoreDir) b.WriteString("}\n\n") accounts, err := ComposeAccounts(sorted) if err != nil { return "", err } b.WriteString(accounts) return b.String(), nil } // ComposeAccounts is the accounts block alone — every user, and nothing about the server. // // **This is the only part of the configuration the mesh writes, and the split is deliberate.** A // server's ports, its TLS paths and its store directory are properties of the container the module // raises: they live in its image and its mounts, and they change when it does. The controller has no // business knowing them, and a controller that did would have to be kept in step with a Dockerfile // it never sees. What only the mesh knows is *who may connect*, so that is what it writes, and the // module's own configuration includes it. // // Four things checked against a running server before this shape was committed to: a user in an // included file authenticates; an unknown user is refused, so the include is the whole authority // rather than an addition to something; a publish outside a user's grant is refused; and rewriting // this file alone and signalling a reload makes a new user appear **without dropping the connection // the mesh already has** — which is what makes every later account, permission or person change cost // nothing (task 1.2's payoff). func ComposeAccounts(principals []Principal) (string, error) { sorted := append([]Principal(nil), principals...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Username() < sorted[j].Username() }) var b strings.Builder b.WriteString("# The mesh's users, composed by the controller. Do not edit: the next\n") b.WriteString("# composition overwrites it. Permissions are derived from what each module\n") b.WriteString("# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\n") // One account for the mesh: accounts in NATS isolate subject spaces entirely, and the mesh is // one space (design 25 §4). The cost of that — that permissions are the only isolation — is // paid in the scoping of every inbox and every ack subject. b.WriteString("accounts {\n MESH {\n users = [\n") for _, p := range sorted { perms, err := PermissionsFor(p) if err != nil { return "", err } if p.PasswordHash == "" { return "", fmt.Errorf("%s has no password hash: a user without one is a user anybody is", p.Username()) } fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", p.Username(), p.PasswordHash) fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish)) fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe)) if perms.AllowResponses { b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n") } b.WriteString(" } }\n") } b.WriteString(" ]\n }\n}\n") return b.String(), nil } func quoted(values []string) string { if len(values) == 0 { return "" } out := make([]string, len(values)) for i, v := range values { out[i] = fmt.Sprintf("%q", v) } return strings.Join(out, ", ") }