Take done keys in from a durable pull consumer, a batch at a time (issue 1945, ADR 0544)
The take-in watched the builder's and the indexer's buckets, started a follower per module that retried on a 30 s timer, and at start took every catalogued module in alphabetically: on 2026-10-25 a restart took in about 2 modules a minute, about 190 followers queued behind 16 slots, and every take-in composed two or three times even with nothing new, so a new build waited 30 minutes and more for its credential. Now two durable consumers named mesh-controller-take-in read the indexer's modules bucket (every module's last trunk value when the consumer is made, then each write) and the builder's builds bucket (each write), at most 16 keys unacknowledged each. The bus keeps what is not taken in: the start's look is the consumers' own catch-up, and a restart costs only what was left unacknowledged. A batch reads the catalogue once, takes each module in once, and asks the composer twice: once for its modules' nodes, once for the bus's node with the grants. A module registered at its trunk whose compositions and grant exist, or that this process took in at the same trunk, composes nothing; a key delivered again is taken in whole. A wait for the builder's or indexer's next write is acknowledged, a wait on a read or a failure refused with a 30 s delay. The backlog is in status, and a backlog standing still for 5 minutes raises take-in-stalled. The controller's account is granted the two consumers' acks. The follower tests go with the followers: twelve tests become three. Research 072: the take-ins on a bounded pull consumer (milestone 4, done keys alone) and the start's look composing only what changed (milestone 2).
This commit is contained in:
@@ -182,3 +182,11 @@ func RaiseAfterDeclarations(r Raiser) error {
|
||||
}
|
||||
return AssertMeshConsumers(r)
|
||||
}
|
||||
|
||||
// TakeInConsumer is the durable consumer the controller takes done keys in from, on the stream of each of TakeInBuckets
|
||||
// (novox/hq ADR 0544 §1).
|
||||
const TakeInConsumer = "mesh-controller-take-in"
|
||||
|
||||
// TakeInBuckets are the buckets the take-in consumes: the indexer's modules (each module's trunk value) and the builder's
|
||||
// builds (each build's key).
|
||||
var TakeInBuckets = []string{delivery.ModulesBucket, delivery.BuilderBuildsBucket}
|
||||
|
||||
@@ -432,6 +432,16 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "carried only at genesis: the controller composes its own permissions into the bus module's composition, " +
|
||||
"and writes a composition's digest only after the bus applied it (novox/hq ADR 0539 clauses 2 and 7)",
|
||||
},
|
||||
{
|
||||
Subject: "$JS.ACK.KV_" + delivery.ModulesBucket + "." + TakeInConsumer + ".>",
|
||||
Reason: "refused: an acknowledgement refused leaves the key delivered again after its wait, and the take-in's first " +
|
||||
"batch with something new composes the bus's node, which grants it (novox/hq ADR 0544 §1)",
|
||||
},
|
||||
{
|
||||
Subject: "$JS.ACK.KV_" + delivery.BuilderBuildsBucket + "." + TakeInConsumer + ".>",
|
||||
Reason: "refused: an acknowledgement refused leaves the key delivered again after its wait, and the take-in's first " +
|
||||
"batch with something new composes the bus's node, which grants it (novox/hq ADR 0544 §1)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + GrantsBucket + ".>",
|
||||
Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " +
|
||||
@@ -707,6 +717,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// ever, refused by the list it already has.
|
||||
sub = append(sub, ControllerFollows...)
|
||||
pub = append(pub, "$JS.ACK.EVENTS."+ControllerName+".>", "$JS.ACK.EVENTS."+MergesConsumerName+".>")
|
||||
// And the take-in's two durable consumers on the indexer's and the builder's buckets (novox/hq ADR 0544 §1):
|
||||
// acknowledged after each key is taken in, and no other consumer of those streams.
|
||||
for _, bucket := range TakeInBuckets {
|
||||
pub = append(pub, "$JS.ACK.KV_"+bucket+"."+TakeInConsumer+".>")
|
||||
}
|
||||
|
||||
// **Where an enrolment's answer goes**, and `allow_responses` does not cover it. That
|
||||
// permits one reply to the reply subject of a message the user received — and a message a
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ accounts {
|
||||
]
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.bucket-deleted", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.ACK.KV_mesh-delivery-builder_builds.mesh-controller-take-in.>", "$JS.ACK.KV_mesh-delivery-indexer_modules.mesh-controller-take-in.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.bucket-deleted", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
Reference in New Issue
Block a user