package broker import ( "fmt" "sort" "strings" ) // The mesh's own streams. // // **These and no more** (novox/hq ADR 0116 task 1.4, as revised by ADR 0118; the two that keep what a // consumer gave up on added for issue 330). An earlier // reading had the controller create *every* stream at genesis, from a fixed set. That is only the // mesh's own half: a seat's streams are created when the module declaring it is registered, and a // module's durable consumers when it is assigned — neither of which has happened at genesis. What // is here is the foundation, which exists before any module does. // // The controller is the only writer of stream definitions (design 25 §3). A module declares // nothing about them and cannot reach the JetStream API to make one. // Retention is how a stream decides what to keep, which is the whole of what distinguishes the // mesh's four relationships on the wire (design 29 §4). type Retention string const ( // RetentionWorkQueue: a message is removed once a consumer acknowledges it. Exactly one // worker does the work, and a worker that dies has its message redelivered. RetentionWorkQueue Retention = "workqueue" // RetentionLastPerSubject: only the newest message on each subject survives. This is the // state shape — a node that was away gets exactly the current declaration and nothing older. RetentionLastPerSubject Retention = "last_per_subject" // RetentionLimits: kept until it ages or the stream fills. Events, where a subscriber that // was down catches up and nobody is obliged to act. RetentionLimits Retention = "limits" ) // A Stream is one of the mesh's own, as the controller asserts it. type Stream struct { Name string Subjects []string Retention Retention // MaxAge in seconds, zero for unbounded. Per stream — JetStream has no per-subject age, // which is why differing retention between modules would mean a stream each. MaxAge int // MaxMsgsPerSubject caps each subject independently, so one noisy emitter cannot push // another's events out of a shared stream. Verified: with a cap of 3, ten messages on one // subject and one on another leave four in the stream, not three. MaxMsgsPerSubject int // Why is carried into the assertion so an operator reading the server's own state finds the // reason there, rather than only in a repository they may not have. Why string // Direct lets a client read a subject's last message without a consumer, which is how a // runtime reads its own membership with no JetStream API beyond one request (ADR 0160). Direct bool // MaxBytes bounds the stream's size, zero for unbounded. With DiscardNew a full stream refuses // what comes next rather than dropping what it holds: the publisher is told, and says so. MaxBytes int64 DiscardNew bool // DuplicatesSeconds is the window in which a message id published twice is kept once; zero for the // server's default (two minutes). DuplicatesSeconds int } // AssignmentsStream holds every assignment's membership, the newest per subject. const AssignmentsStream = "ASSIGNMENTS" // What a durable consumer gave up on is kept (novox/hq issue 330, design 25 §3). // // **The server says it and keeps it; the controller copies it.** A consumer that handed a message over // as often as it may stops offering it and publishes `$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES` with // the stream and the message's sequence. DeadLetterNoticesStream captures those advisories as the // server publishes them, so one said while no controller listens is still there when one starts. The // controller's consumer on it fetches the given-up message by its sequence, while the source stream // still holds it, and keeps a copy in DeadLettersStream under DeadLetterSubject, with the consumer, // the subject, how often it was handed over and when it was given up. It stays there until a person // delivers it again or drops it, with why; a condition is open for as long as it does. const ( DeadLetterNoticesStream = "DEAD_LETTER_NOTICES" DeadLettersStream = "DEAD_LETTERS" // MaxDeliveriesAdvisories is the subject the server says a given-up message on. MaxDeliveriesAdvisories = "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>" deadLetterPrefix = "mesh.events.dead." againPrefix = "mesh.again." // DeadLettersBytes bounds the kept copies. Full, the stream refuses the next copy, which is said // as the consumer's condition; it never drops one it holds. DeadLettersBytes = 256 << 20 ) // DeadLetterSubject is where a message one consumer gave up on is kept: `mesh.events.dead..`. func DeadLetterSubject(stream, consumer string) string { return deadLetterPrefix + stream + "." + consumer } // DeadLetterOf is the stream and consumer a kept message's subject names; false for any other subject. func DeadLetterOf(subject string) (stream, consumer string, ok bool) { rest, found := strings.CutPrefix(subject, deadLetterPrefix) if !found { return "", "", false } stream, consumer, ok = strings.Cut(rest, ".") return stream, consumer, ok && stream != "" && consumer != "" && !strings.Contains(consumer, ".") } // AgainSubject is where an event given up on is delivered again to the one consumer that gave it up, // and to nobody else: `mesh.again..` and the original subject without its `mesh.`. Every // consumer on EVENTS filters its own (AgainFilter), and a module's runtime reads the event's key from // the tokens around `.event.`, so the handler sees the same key it saw the first time. func AgainSubject(consumer, original string) string { return againPrefix + consumer + "." + strings.TrimPrefix(original, "mesh.") } // AgainFilter is the one consumer's share of the subjects events are delivered again on. func AgainFilter(consumer string) string { return againPrefix + consumer + ".>" } // OriginalOfAgain is the subject an event delivered again was first published on; false for a subject // that is not one delivered again. func OriginalOfAgain(subject string) (string, bool) { rest, found := strings.CutPrefix(subject, againPrefix) if !found { return "", false } _, original, ok := strings.Cut(rest, ".") if !ok || original == "" { return "", false } return "mesh." + original, true } // MeshStreams is the foundation set, in the order a person reads it. // // **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live // under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a // stream of them is a stream of the least valuable messages the mesh sends, competing for the // same retention as the ones that matter. // // **EVENTS filters on the `event` token**, which is the reason that token exists. A module's // namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist // every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps // tools on core NATS, where a lost call is a timeout the caller already handles). func MeshStreams() []Stream { return []Stream{ { Name: "CONTROL", // A build's outcome is no longer here: it is the build-machine seat's own event, so one // publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121). Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, Retention: RetentionWorkQueue, Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " + "store is away and the message is redelivered; nothing is dropped", }, { Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, Retention: RetentionLastPerSubject, Why: "one declaration per node, always the newest; a node that sees sequence n refuses " + "n-1 by construction (issue 107)", }, { Name: AssignmentsStream, Subjects: []string{"mesh.assignment.*.*"}, Retention: RetentionLastPerSubject, Direct: true, Why: "one membership per assignment, always the newest: what the mesh issued this module " + "on this machine to serve and to reach (ADR 0160); read directly by the runtime it is for", }, { Name: EventsStream, // A seat's own events ride here too: they are 1:many like any event, and the // `event` token keeps them clear of both the seat's work queue (`accept`) and its // tools (`tool`), which must not be persisted. // And an event given up on, delivered again to the one consumer that gave it up // (novox/hq issue 330): under `mesh.again..`, which only that consumer filters. Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>", againPrefix + ">"}, Retention: RetentionLimits, MaxAge: 7 * 24 * 60 * 60, MaxMsgsPerSubject: 10000, Why: "a subscriber that was down catches up; tool traffic under the same prefix is " + "excluded by the event token; per-subject caps keep a noisy emitter from " + "evicting a quiet one without splitting the stream", }, { Name: DeadLetterNoticesStream, Subjects: []string{MaxDeliveriesAdvisories}, Retention: RetentionWorkQueue, MaxAge: 7 * 24 * 60 * 60, Why: "the server's word that a consumer gave up on a message, kept until the controller has " + "copied the message into DEAD_LETTERS (novox/hq issue 330); a week, the longest the source " + "streams keep what they are about", }, { Name: DeadLettersStream, Subjects: []string{deadLetterPrefix + ">"}, Retention: RetentionLimits, MaxBytes: DeadLettersBytes, DiscardNew: true, DuplicatesSeconds: 24 * 60 * 60, Why: "every message a consumer gave up on, with its consumer, subject, deliveries and when, kept " + "until a person delivers it again or drops it with why (novox/hq issue 330); no age, and full " + "it refuses the next copy rather than drop one it holds", }, } } // DeliverSubjectFor is where a push consumer's messages land. // // **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push // consumer delivers onto an ordinary subject, and everything subscribed to that subject gets a // copy. The controller holds a consumer called `controller` on CONTROL and another called // `controller` on EVENTS; while both were given `_DELIVER.controller`, the one process holding // both subscriptions acted on every message twice — a joining machine was enrolled twice from one // request, and the second enrolment minted a credential that replaced the one the machine had just // been handed. Every report and every followed event doubled the same way, silently: nothing is // redelivered, no count is wrong, the work simply happens twice. // // The stream belongs in it because the pair is what identifies a consumer — the server scopes a // durable's name to its stream, and this subject was the one place that scoping was dropped. It // stays inside what a controller may already subscribe (`_DELIVER.controller.>`). func DeliverSubjectFor(c Consumer) string { return "_DELIVER." + c.Name + "." + c.Stream } // An Asserter is the part of a JetStream connection stream assertion needs. Narrow on purpose: it // keeps this testable without a server, and keeps the client library out of everything that only // wants to know what the streams are. type Asserter interface { // EnsureStream creates the stream if absent and updates it to match if present. It must be // idempotent: the controller asserts on every start, not only at genesis. EnsureStream(s Stream) error } // AssertMeshStreams brings the foundation set into being, in order, and says which one failed // rather than that something did. // // Asserted on every start rather than created once at genesis, because a stream that was deleted, // or a mesh raised from a restored backup, must converge rather than run without the guarantee // its messages assume. Idempotence is the whole requirement. func AssertMeshStreams(a Asserter) error { for _, s := range MeshStreams() { if err := a.EnsureStream(s); err != nil { return fmt.Errorf("asserting stream %s: %w", s.Name, err) } } return nil } // Overlaps reports subject filters claimed by more than one stream. // // **Corrected against the server**: an earlier version of this comment said NATS accepts // overlapping streams and stores the message twice. It does not — it refuses the second stream // with "subjects overlap with an existing stream" (verified against nats-server 2.10). The check // still earns its place, for a different reason: the server's refusal arrives when the controller // is applying, naming one stream, at a moment when the mesh is half-configured. This one arrives // where the set is written, names both, and cannot reach a running mesh. // // It also decides a design question. Because overlap is refused rather than merged, a shared // EVENTS stream and a per-module stream cannot coexist — the module's would be refused — so // "one stream for most, its own for a module that wants different retention" is not an option // the server allows. It is all of one or all of the other. func Overlaps() []string { seen := map[string]string{} var clashes []string for _, s := range MeshStreams() { for _, subject := range s.Subjects { if first, ok := seen[subject]; ok { clashes = append(clashes, fmt.Sprintf("%s and %s both claim %s", first, s.Name, subject)) continue } seen[subject] = s.Name } } sort.Strings(clashes) return clashes } // The mesh's own consumers. // // A seat's streams and a module's consumers are derived from declarations (derived.go). These two // are not: **the controller is not a module and files no manifest**, so its authority and its // subscriptions cannot come from a declaration that does not exist. They are named here, where the // mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would // hear every event in the mesh, which it has no business doing and which would make its permission // list stop explaining anything. // ControllerName is the controller's durable consumer on each stream it reads, and the name its // ack subject is derived from (nats.go: `$JS.ACK..controller.>`). const ControllerName = "controller" // ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states // under it (novox/hq ADR 0134). // // **Written here as well as in `link`, and a test keeps them agreeing.** `link` imports `broker`, so // `broker` cannot import `link`; a grant naming a subject the controller never publishes is authority // nobody uses, and a controller publishing one the grant omits is refused at the moment it has // something to say. const ControllerSeat = "mesh-controller" var ControllerStates = []string{"applied", "refused", "built-before", // What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in // severity, resolver or silence, and cleared. The operator-channel's holder and any other surface // consume them; the controller tells nobody itself. "condition-raised", "condition-changed", "condition-cleared", // And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a // machine that is not the control node, so the controller going quiet is itself said. "doctor-heartbeat", // And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228). "secret-replaced", // And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh // made by itself is said like one a person made, never quietly. "healer-acted", // And a build put back after its gate failed on its first machine (novox/hq ADR 0236, to-be 45 §8). "rolled-back", // And a pull request's merge check, judged (novox/hq to-be 45 §9): what the forge's holder sets as // the pull request's status, and anybody else may read. "checked", // And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from. "plan-moved"} // BusAdvisories are what the bus server says about the mesh's own account that the controller // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it // may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to // publish, and the controller's subscription changes nothing on the bus. var BusAdvisories = []string{ "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", } // ControllerFollows are the events the controller reacts to: the catalogue saying a module's // current version moved, and a catalogue that has just started saying it may have missed builds. // // **Derived the same way a module's subscription is**, from the emitter and the bare local event // name, rather than written out. They were written out while the catalogue still spelled its events // as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a // hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from // the other side. Deriving them means the conversion could not leave these behind. var ControllerFollows = []string{ moduleEventSubject("mesh-catalog", "upgraded"), moduleEventSubject("mesh-catalog", "catching-up"), // A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a // message on the control branch. Same three audiences, one publish: whoever asked, this, and the // catalogue. seatEventSubject("node-build-agent", "built"), // The forge's merges: what moved a source, so the mesh builds what that source produces // without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name. moduleEventSubject("gitea", "pull.merged"), // The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one // build machine keeps answering on its seat until build-agent replaces it, and the outcome that // registers build-agent itself comes from there. Appended, for the same reason as above; goes // with the retired seat row. seatEventSubject("mesh-build-machine", "built"), // **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and // that consumer recovered. The one pattern on this list, and a narrow one — two named events, // from whichever module provides — because the rule is about every provider, and a list of // providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider // failed every consumer for a day and only its journal said so (issue 179). Appended, because // the index is a name. moduleEventSubject("*", ProvisionerFailing), moduleEventSubject("*", ProvisionerRecovered), // **What becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired, waiting // for a person, re-enabled, deleted — a provider's third word, from whichever module provides. // Appended, because the index is a name. moduleEventSubject("*", ProvisionerRetirement), // **A pull request's head, announced** (novox/hq to-be 45 §9): what the controller asks the build // seat to check before it merges — every machine of the facts snapshot composed with the change. // Appended, because the index is a name. moduleEventSubject("gitea", "pull.updated"), } // The provider standing events, by their local names. Written here as well as in the catalogue // (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing. const ( ProvisionerFailing = "provisioner.failing" ProvisionerRecovered = "provisioner.recovered" ProvisionerRetirement = "provisioner.retirement" ) // moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so // what the controller subscribes and what the emitter is permitted to publish cannot drift apart. func moduleEventSubject(module, event string) string { return "mesh.mod." + module + ".event." + event } // seatEventSubject is where a role's own event lands, derived the same way a holder's permission is. func seatEventSubject(seat, verb string) string { return "mesh.seat." + seat + ".event." + verb } // EventsStream holds every module's and every role's events, a build's log among them. const EventsStream = "EVENTS" // MeshConsumers is what the controller consumes, in the order a person reads it. // // **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's, // not the server's (window.go): a message is held with a nak-and-delay until the controller either // takes it or gives up and says so. A max-deliver here would give up on a push that was being // held through a store restart — the exact message the stream exists to protect — some minutes // before the controller had finished deciding about it. func MeshConsumers() []Consumer { return []Consumer{ { Name: ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 30, Why: "the controller is the single consumer of what nodes say; explicit ack and no " + "max-deliver, because the store window's bound is the controller's own", }, { Name: ControllerName, Stream: "EVENTS", Filters: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName)), Push: true, AckWaitSeconds: 30, MaxDeliver: 5, // One at a time (novox/hq issue 175): acting on a merge builds for minutes, and an // announcement handed over behind it must wait on the server, not time out on the // client and come back to be acted on again. MaxAckPending: 1, FromNow: true, // **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week // behind once and held every merge after it. What a reset drops is caught up elsewhere — // a merge by the catch-up pass that reads the forge (issue 266), a build's outcome from the // build records a plan settles from (issue 214), a provider's failing word by the provider // saying it again every quarter of an hour (ADR 0224). Resettable: "what it drops is caught up: merges by the catch-up pass (issue 266), build outcomes " + "from the build records (issue 214), a provider's failing word said again (ADR 0224)", Why: "the events the mesh's own controller reacts to, one at a time; after " + "max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " + "a person delivers it again or drops it", }, // What the server said a consumer gave up on (novox/hq issue 330): copied into DEAD_LETTERS and // acknowledged. No max-deliver: a notice the controller could not copy is offered again, and said. { Name: ControllerName, Stream: DeadLetterNoticesStream, Push: true, AckWaitSeconds: 30, Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " + "because a notice it gave up on would lose the message it is about", }, } } // NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330). func NoticesConsumer() Consumer { for _, c := range MeshConsumers() { if c.Stream == DeadLetterNoticesStream { return c } } panic("the mesh's consumers carry none on " + DeadLetterNoticesStream) } // AssertServingConsumers are the controller's own consumers its serving cannot go without: all but the // one on DEAD_LETTER_NOTICES, which the keeper of dead letters asserts and retries by itself, so a fault // there never stops the controller serving (novox/hq issue 330). func AssertServingConsumers(e Ensurer) error { for _, c := range MeshConsumers() { if c.Stream == DeadLetterNoticesStream { continue } if err := e.EnsureConsumer(c); err != nil { return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err) } } return nil } // Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason // Asserter is. type Ensurer interface { EnsureConsumer(c Consumer) error } // AssertMeshConsumers brings the controller's own consumers into being, and says which one failed. // // After the streams, necessarily: a consumer on a stream that does not exist is refused, and the // refusal names the stream rather than the order. func AssertMeshConsumers(e Ensurer) error { for _, c := range MeshConsumers() { if err := e.EnsureConsumer(c); err != nil { return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err) } } return nil }