|
|
|
@@ -144,26 +144,14 @@ func OriginalOfAgain(subject string) (string, bool) {
|
|
|
|
|
|
|
|
|
|
// MeshStreams is the foundation set, in the order a person reads it.
|
|
|
|
|
//
|
|
|
|
|
// **CONTROL names its subjects rather than taking `mesh.control.>`**, because heartbeats live
|
|
|
|
|
// under that prefix and must not be persisted: a lost heartbeat is the next heartbeat, and a
|
|
|
|
|
// stream of them is a stream of the least valuable messages the mesh sends, competing for the
|
|
|
|
|
// same retention as the ones that matter.
|
|
|
|
|
//
|
|
|
|
|
// **EVENTS filters on the `event` token**, which is the reason that token exists. A module's
|
|
|
|
|
// namespace carries both its events and its tool calls; a filter of `mesh.mod.*.>` would persist
|
|
|
|
|
// every tool invocation in the mesh, and a tool call must never be persisted (design 25 §3 keeps
|
|
|
|
|
// tools on core NATS, where a lost call is a timeout the caller already handles).
|
|
|
|
|
func MeshStreams() []Stream {
|
|
|
|
|
return []Stream{
|
|
|
|
|
{
|
|
|
|
|
Name: "CONTROL",
|
|
|
|
|
// A build's outcome is no longer here: it is the build-machine seat's own event, so one
|
|
|
|
|
// publish reaches whoever asked, the controller and the catalogue (novox/hq ADR 0121).
|
|
|
|
|
Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
|
|
|
|
|
Retention: RetentionWorkQueue,
|
|
|
|
|
Why: "the store-window guarantee (ADR 0083): the controller naks with a delay while its " +
|
|
|
|
|
"store is away and the message is redelivered; nothing is dropped",
|
|
|
|
|
},
|
|
|
|
|
// CONTROL, the ended controller's work queue of what nodes reported, is no longer the mesh's (novox/hq ADR 0590
|
|
|
|
|
// clause 6): not declared here, which would make it again; the bus module's tools remove it.
|
|
|
|
|
{
|
|
|
|
|
Name: "NODES",
|
|
|
|
|
Subjects: []string{"mesh.node.*.declare"},
|
|
|
|
@@ -235,18 +223,6 @@ const ProvAppliedStream = "PROV_APPLIED"
|
|
|
|
|
// all (ProvisionerAppliedSubject).
|
|
|
|
|
const ProvAppliedSubjects = "prov.applied.>"
|
|
|
|
|
|
|
|
|
|
// eventsTheControllerFollows are the events the controller's events consumer takes: all it follows but the forge's
|
|
|
|
|
// merges, which the merges consumer takes (MergesConsumerName).
|
|
|
|
|
func eventsTheControllerFollows() []string {
|
|
|
|
|
var out []string
|
|
|
|
|
for _, subject := range ControllerFollows {
|
|
|
|
|
if subject != ControllerFollows[3] {
|
|
|
|
|
out = append(out, subject)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return append(out, ControllerHears...)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// DeliverSubjectFor is where a push consumer's messages land.
|
|
|
|
|
//
|
|
|
|
|
// **Per consumer, which means per stream as well as per name** (novox/hq 04-ISSUES/146). A push
|
|
|
|
@@ -321,147 +297,17 @@ func Overlaps() []string {
|
|
|
|
|
return clashes
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The mesh's own consumers.
|
|
|
|
|
//
|
|
|
|
|
// A seat's streams and a module's consumers are derived from declarations (derived.go). These two
|
|
|
|
|
// are not: **the controller is not a module and files no manifest**, so its authority and its
|
|
|
|
|
// subscriptions cannot come from a declaration that does not exist. They are named here, where the
|
|
|
|
|
// mesh's own streams are named, and narrowly — a controller subscribing `mesh.mod.*.event.>` would
|
|
|
|
|
// hear every event in the mesh, which it has no business doing and which would make its permission
|
|
|
|
|
// list stop explaining anything.
|
|
|
|
|
|
|
|
|
|
// ControllerName is the controller's durable consumer on each stream it reads, and the name its
|
|
|
|
|
// ack subject is derived from (nats.go: `$JS.ACK.<stream>.controller.>`).
|
|
|
|
|
const ControllerName = "controller"
|
|
|
|
|
|
|
|
|
|
// MergesConsumerName is the controller's durable consumer of the forge's merged event on EVENTS, apart from the
|
|
|
|
|
// events consumer (novox/hq ADR 0407 decision 3, issue 973). A merge is never dropped, so this consumer has no
|
|
|
|
|
// max-deliver, which would send a merge held five times to DEAD_LETTERS to wait for a person, and no healer
|
|
|
|
|
// resets it, which would drop the merges it holds. A consumer of its own because both of those are properties of
|
|
|
|
|
// a consumer, not of a subject.
|
|
|
|
|
const MergesConsumerName = "controller-merges"
|
|
|
|
|
|
|
|
|
|
// ControllerSeat is the role the control plane holds, and ControllerStates are the facts it states
|
|
|
|
|
// under it (novox/hq ADR 0134).
|
|
|
|
|
//
|
|
|
|
|
// **Written here as well as in `link`, and a test keeps them agreeing.** `link` imports `broker`, so
|
|
|
|
|
// `broker` cannot import `link`; a grant naming a subject the controller never publishes is authority
|
|
|
|
|
// nobody uses, and a controller publishing one the grant omits is refused at the moment it has
|
|
|
|
|
// something to say.
|
|
|
|
|
const ControllerSeat = "mesh-controller"
|
|
|
|
|
|
|
|
|
|
var ControllerStates = []string{"applied", "refused", "built-before",
|
|
|
|
|
// What is wrong, said as it changes (novox/hq to-be 45 §2): a condition raised, changed in
|
|
|
|
|
// severity, resolver or silence, and cleared. The operator-channel's holder and any other surface
|
|
|
|
|
// consume them; the controller tells nobody itself.
|
|
|
|
|
"condition-raised", "condition-changed", "condition-cleared",
|
|
|
|
|
// And the self-check's heartbeat, at the end of every run (to-be 45 §4, S10): watched from a
|
|
|
|
|
// machine that is not the control node, so the controller going quiet is itself said.
|
|
|
|
|
"doctor-heartbeat",
|
|
|
|
|
// And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228).
|
|
|
|
|
"secret-replaced",
|
|
|
|
|
// And every act a healer takes on a condition (novox/hq to-be 45 §7, Phase 3): a repair the mesh
|
|
|
|
|
// made by itself is said like one a person made, never quietly.
|
|
|
|
|
"healer-acted",
|
|
|
|
|
// And a build put back after its gate failed on its first machine (novox/hq ADR 0236, to-be 45 §8).
|
|
|
|
|
"rolled-back",
|
|
|
|
|
// And a pull request's merge check, judged (novox/hq to-be 45 §9): what the forge's holder sets as
|
|
|
|
|
// the pull request's status, and anybody else may read.
|
|
|
|
|
"checked",
|
|
|
|
|
// (A walk kept in a new state and a tier sent to a machine are gone with the plans, novox/hq ADR 0539.)
|
|
|
|
|
// And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and
|
|
|
|
|
// each registered module's declared config (design 57 F3).
|
|
|
|
|
"assignment-changed", "declared-config-changed", "predefined-changed",
|
|
|
|
|
// And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937).
|
|
|
|
|
"artifacts-gone",
|
|
|
|
|
// And a bucket of the plans' state deleted at the controller's first start, and the declarations' stream once no node
|
|
|
|
|
// reads one (novox/hq ADR 0539 clauses 11 and 13).
|
|
|
|
|
"bucket-deleted"}
|
|
|
|
|
|
|
|
|
|
// BusAdvisories are what the bus server says about the mesh's own account that the controller
|
|
|
|
|
// reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it
|
|
|
|
|
// may and gave up on it, and one that was deleted. Read-only: an advisory is the server's to
|
|
|
|
|
// publish, and the controller's subscription changes nothing on the bus.
|
|
|
|
|
var BusAdvisories = []string{
|
|
|
|
|
"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>",
|
|
|
|
|
"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ControllerFollows are the events the controller reacts to: the catalogue saying a module's
|
|
|
|
|
// current version moved, and a catalogue that has just started saying it may have missed builds.
|
|
|
|
|
//
|
|
|
|
|
// **Derived the same way a module's subscription is**, from the emitter and the bare local event
|
|
|
|
|
// name, rather than written out. They were written out while the catalogue still spelled its events
|
|
|
|
|
// as the old bus's routing keys, and the moment those were converted (novox/hq 04-ISSUES/127) a
|
|
|
|
|
// hard-coded pair became a controller listening to a subject nothing publishes — the same fault, from
|
|
|
|
|
// the other side. Deriving them means the conversion could not leave these behind.
|
|
|
|
|
var ControllerFollows = []string{
|
|
|
|
|
moduleEventSubject("mesh-catalog", "upgraded"),
|
|
|
|
|
moduleEventSubject("mesh-catalog", "catching-up"),
|
|
|
|
|
// A build's outcome, which is the build-machine role's own event now (ADR 0121) rather than a
|
|
|
|
|
// message on the control branch. Same three audiences, one publish: whoever asked, this, and the
|
|
|
|
|
// catalogue.
|
|
|
|
|
seatEventSubject("node-build-agent", "built"),
|
|
|
|
|
// The forge's merges: what moved a source, so the mesh builds what that source produces
|
|
|
|
|
// without anybody telling it (novox/hq 04-ISSUES/131). Appended, because the index is a name.
|
|
|
|
|
moduleEventSubject("gitea", "pull.merged"),
|
|
|
|
|
// The retired build role's outcome too, while the handover runs (novox/hq ADR 0190): the one
|
|
|
|
|
// build machine keeps answering on its seat until build-agent replaces it, and the outcome that
|
|
|
|
|
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
|
|
|
|
// with the retired seat row.
|
|
|
|
|
seatEventSubject("mesh-build-machine", "built"),
|
|
|
|
|
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
|
|
|
|
|
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
|
|
|
|
|
// from whichever module provides — because the rule is about every provider, and a list of
|
|
|
|
|
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
|
|
|
|
|
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
|
|
|
|
|
// the index is a name.
|
|
|
|
|
moduleEventSubject("*", ProvisionerFailing),
|
|
|
|
|
moduleEventSubject("*", ProvisionerRecovered),
|
|
|
|
|
// **What becomes of a consumer the mesh stopped asking for** (novox/hq ADR 0230): retired, waiting
|
|
|
|
|
// for a person, re-enabled, deleted — a provider's third word, from whichever module provides.
|
|
|
|
|
// Appended, because the index is a name.
|
|
|
|
|
moduleEventSubject("*", ProvisionerRetirement),
|
|
|
|
|
// **A pull request's head, announced** (novox/hq to-be 45 §9): what the controller asks the build
|
|
|
|
|
// seat to check before it merges — every machine of the facts snapshot composed with the change.
|
|
|
|
|
// Appended, because the index is a name.
|
|
|
|
|
moduleEventSubject("gitea", "pull.updated"),
|
|
|
|
|
// **The operator's answers to what the controller asked** (novox/hq ADR 0259): the router's warrant, or
|
|
|
|
|
// the end of an ask without one, said to the controller alone under its own name. On the stream, so a
|
|
|
|
|
// controller that was away hears what was decided meanwhile. Appended, because the index is a name.
|
|
|
|
|
DecidedSubject,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ControllerHears are events the controller's events consumer takes beside the ones it follows, and that need no
|
|
|
|
|
// subscribe grant of their own. **A push consumer delivers onto its deliver subject** (`_DELIVER.controller.>`, already
|
|
|
|
|
// granted), so the subject a consumer filters on is never one the controller subscribes: listing this one in
|
|
|
|
|
// ControllerFollows would add a subscribe grant the installer's carried user list, which lives in mesh-host, would
|
|
|
|
|
// have to carry too (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), for nothing the consumer uses.
|
|
|
|
|
//
|
|
|
|
|
// A merge mesh-delivery made, announced (novox/hq ADR 0420 §7, issue 1025): with the ask whose Approve it merged on and
|
|
|
|
|
// the heads that Approve covered, kept a week to read the warrant a moved module's registration needs.
|
|
|
|
|
var ControllerHears = []string{DeliveryMergedSubject}
|
|
|
|
|
|
|
|
|
|
// DeliveryMergedEvent is the event a module says when it merged a pull request.
|
|
|
|
|
const DeliveryMergedEvent = "merged"
|
|
|
|
|
|
|
|
|
|
// DeliveryMergedSubject is where the delivery's owner says it merged a pull request: one wildcard subject, the
|
|
|
|
|
// module that claims the delivery seat being whichever the catalogue says (novox/hq ADR 0437 §3). Who said it is
|
|
|
|
|
// read from the subject, and whether it claims the seat is the controller's to check.
|
|
|
|
|
var DeliveryMergedSubject = moduleEventSubject("*", DeliveryMergedEvent)
|
|
|
|
|
|
|
|
|
|
// DeliveryMergedSubjectOf is where one module says it merged a pull request.
|
|
|
|
|
func DeliveryMergedSubjectOf(module string) string {
|
|
|
|
|
return moduleEventSubject(module, DeliveryMergedEvent)
|
|
|
|
|
}
|
|
|
|
|
// The ended controller's consumers' names (novox/hq ADR 0590 clause 6): kept only so its leftovers are named for their
|
|
|
|
|
// removal (ControllerLeftoverConsumers) and the watchman can read whether `controller` on DEAD_LETTER_NOTICES still
|
|
|
|
|
// stands.
|
|
|
|
|
const (
|
|
|
|
|
ControllerName = "controller"
|
|
|
|
|
MergesConsumerName = "controller-merges"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// AsksSeat is the seat an ask is made on and its warrant heard from (novox/hq ADR 0259): the router's.
|
|
|
|
|
const AsksSeat = "operator-channel"
|
|
|
|
|
|
|
|
|
|
// DecidedSubject is where the router says the controller's warrants: the seat's event named by the
|
|
|
|
|
// controller as its caller.
|
|
|
|
|
var DecidedSubject = seatEventSubject(AsksSeat, "decided."+ControllerSeat)
|
|
|
|
|
|
|
|
|
|
// The provider standing events, by their local names. Written here as well as in the catalogue
|
|
|
|
|
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
|
|
|
|
const (
|
|
|
|
@@ -484,108 +330,9 @@ func seatEventSubject(seat, verb string) string {
|
|
|
|
|
// EventsStream holds every module's and every role's events, a build's log among them.
|
|
|
|
|
const EventsStream = "EVENTS"
|
|
|
|
|
|
|
|
|
|
// MeshConsumers is what the controller consumes, in the order a person reads it.
|
|
|
|
|
//
|
|
|
|
|
// **Unlimited redelivery on CONTROL, deliberately.** The store window's bound is the controller's,
|
|
|
|
|
// not the server's (window.go): a message is held with a nak-and-delay until the controller either
|
|
|
|
|
// takes it or gives up and says so. A max-deliver here would give up on a push that was being
|
|
|
|
|
// held through a store restart — the exact message the stream exists to protect — some minutes
|
|
|
|
|
// before the controller had finished deciding about it.
|
|
|
|
|
func MeshConsumers() []Consumer {
|
|
|
|
|
return ControllerConsumers()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ControllerConsumers are the consumers the controller's own event loop takes from: MeshConsumers without the build
|
|
|
|
|
// seat's check workers, which the builders take from and the loop never does (issue 1063). A consumer added to the
|
|
|
|
|
// controller is counted here without a change elsewhere.
|
|
|
|
|
func ControllerConsumers() []Consumer {
|
|
|
|
|
return []Consumer{
|
|
|
|
|
{
|
|
|
|
|
Name: ControllerName,
|
|
|
|
|
Stream: "CONTROL",
|
|
|
|
|
Push: true,
|
|
|
|
|
AckWaitSeconds: 30,
|
|
|
|
|
Why: "the controller is the single consumer of what nodes say; explicit ack and no " +
|
|
|
|
|
"max-deliver, because the store window's bound is the controller's own",
|
|
|
|
|
},
|
|
|
|
|
// The forge's merges (novox/hq ADR 0407 decision 3, issue 973): never dropped, so no max-deliver and no
|
|
|
|
|
// reset; a merge the controller cannot record yet is held with a growing delay until it can. **Asserted before
|
|
|
|
|
// the events consumer**, which stops filtering merges: the order of this list is the order of assertion.
|
|
|
|
|
{
|
|
|
|
|
Name: MergesConsumerName,
|
|
|
|
|
Stream: "EVENTS",
|
|
|
|
|
// Within what the controller may subscribe already (`_DELIVER.controller.>`).
|
|
|
|
|
DeliverSubject: "_DELIVER." + ControllerName + ".merges",
|
|
|
|
|
Filters: []string{ControllerFollows[3]},
|
|
|
|
|
Push: true,
|
|
|
|
|
AckWaitSeconds: 30,
|
|
|
|
|
MaxAckPending: 1,
|
|
|
|
|
FromNow: true,
|
|
|
|
|
TakesOverFrom: ControllerName,
|
|
|
|
|
Why: "the forge's merges, one at a time; no max-deliver and never reset, because a merge is never " +
|
|
|
|
|
"dropped and no person releases one",
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
Name: ControllerName,
|
|
|
|
|
Stream: "EVENTS",
|
|
|
|
|
Filters: append(eventsTheControllerFollows(), AgainFilter(ControllerName)),
|
|
|
|
|
Push: true,
|
|
|
|
|
AckWaitSeconds: 30,
|
|
|
|
|
MaxDeliver: 5,
|
|
|
|
|
// One at a time (novox/hq issue 175): acting on a merge builds for minutes, and an
|
|
|
|
|
// announcement handed over behind it must wait on the server, not time out on the
|
|
|
|
|
// client and come back to be acted on again.
|
|
|
|
|
MaxAckPending: 1,
|
|
|
|
|
FromNow: true,
|
|
|
|
|
// **The one consumer the mesh resets by itself** (healer H4, issue 248): it fell a week
|
|
|
|
|
// behind once and held every merge after it. What a reset drops is caught up elsewhere — a
|
|
|
|
|
// build's outcome from the build records a plan settles from (issue 214), a provider's failing
|
|
|
|
|
// word by the provider saying it again every quarter of an hour (ADR 0224). **Not the merges**:
|
|
|
|
|
// they are the merges consumer's below, which a reset leaves alone (ADR 0407 decision 3).
|
|
|
|
|
Resettable: "what it drops is caught up: build outcomes from the build records (issue 214), a " +
|
|
|
|
|
"provider's failing word said again (ADR 0224); the forge's merges are not its, they have a " +
|
|
|
|
|
"consumer of their own that no reset touches",
|
|
|
|
|
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
|
|
|
|
"max-deliver it gives the event up, and the controller keeps it in DEAD_LETTERS until " +
|
|
|
|
|
"a person delivers it again or drops it",
|
|
|
|
|
},
|
|
|
|
|
// What the server said a consumer gave up on (novox/hq issue 330): copied into DEAD_LETTERS and
|
|
|
|
|
// acknowledged. No max-deliver: a notice the controller could not copy is offered again, and said.
|
|
|
|
|
{
|
|
|
|
|
Name: ControllerName,
|
|
|
|
|
Stream: DeadLetterNoticesStream,
|
|
|
|
|
Push: true,
|
|
|
|
|
AckWaitSeconds: 30,
|
|
|
|
|
Why: "the controller copies each message a consumer gave up on into DEAD_LETTERS; no max-deliver, " +
|
|
|
|
|
"because a notice it gave up on would lose the message it is about",
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NoticesConsumer is the controller's consumer on DEAD_LETTER_NOTICES (novox/hq issue 330).
|
|
|
|
|
func NoticesConsumer() Consumer {
|
|
|
|
|
for _, c := range MeshConsumers() {
|
|
|
|
|
if c.Stream == DeadLetterNoticesStream {
|
|
|
|
|
return c
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
panic("the mesh's consumers carry none on " + DeadLetterNoticesStream)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// AssertServingConsumers are the controller's own consumers its serving cannot go without: all but the
|
|
|
|
|
// one on DEAD_LETTER_NOTICES, which the keeper of dead letters asserts and retries by itself, so a fault
|
|
|
|
|
// there never stops the controller serving (novox/hq issue 330).
|
|
|
|
|
func AssertServingConsumers(e Ensurer) error {
|
|
|
|
|
for _, c := range MeshConsumers() {
|
|
|
|
|
if c.Stream == DeadLetterNoticesStream {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if err := e.EnsureConsumer(c); err != nil {
|
|
|
|
|
return fmt.Errorf("asserting consumer %s on %s: %w", c.Name, c.Stream, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
// MeshConsumers are the mesh's own durable consumers, beside those derived from declarations: none since the controller
|
|
|
|
|
// ended (novox/hq ADR 0590), whose consumers they were.
|
|
|
|
|
func MeshConsumers() []Consumer { return nil }
|
|
|
|
|
|
|
|
|
|
// Ensurer is the part of a JetStream connection consumer assertion needs, narrow for the reason
|
|
|
|
|
// Asserter is.
|
|
|
|
@@ -593,7 +340,7 @@ type Ensurer interface {
|
|
|
|
|
EnsureConsumer(c Consumer) error
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// AssertMeshConsumers brings the controller's own consumers into being, and says which one failed.
|
|
|
|
|
// AssertMeshConsumers brings the mesh's own consumers into being, and says which one failed.
|
|
|
|
|
//
|
|
|
|
|
// After the streams, necessarily: a consumer on a stream that does not exist is refused, and the
|
|
|
|
|
// refusal names the stream rather than the order.
|
|
|
|
|