novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

issue 974: the builders say the checks they run in a bucket that ages, and a builder drops a waiting request for a head and trunk commit another is running

Browse Source
This commit is contained in:
jschoubben
2026-10-17 14:10:26 +02:00
parent 357e08a4f6
commit bd257afbd5
10 changed files with 380 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/broker/checks.go
+6
View File
@@ -92,6 +92,12 @@ func holderCheckGrants() []string {
"$JS.API.CONSUMER.MSG.NEXT."+w.Stream+"."+w.Name,
"$JS.ACK."+w.Stream+"."+w.Name+".>")
}
// And the bucket of the checks they are running: each writes the key of its own check, reads another's, and the bucket
// is a controller's own bucket the build seat's holders write by design (ADR 0406 §3).
pub = append(pub,
"$KV."+CheckRunningBucket+".>",
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
return pub
}
internal/broker/checks_test.go
+3
View File
@@ -26,6 +26,9 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_READY.CHECKS_READY_worker",
"$JS.API.CONSUMER.MSG.NEXT.CHECKS_OTHER.CHECKS_OTHER_worker",
"$JS.ACK.CHECKS_OTHER.CHECKS_OTHER_worker.>",
// And the bucket of the checks running, which it writes and reads (ADR 0406 §3).
"$KV." + CheckRunningBucket + ".>",
"$JS.API.DIRECT.GET.KV_" + CheckRunningBucket + ".>",
} {
has(t, perms.Publish, s)
}
internal/broker/controller_buckets.go
+28 -2
View File
@@ -49,6 +49,13 @@ var (
// controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease
// holder alone.
RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent")
// CheckRunningBucket holds the pull request checks the builders are running now (novox/hq ADR 0406 §3, issue 974): per
// pull request a key `<owner>.<repo>.<number>` (or `group.<name>`) holding the head and trunk commit being checked, the
// builder and its build. A builder puts it when it starts a check and again on each heartbeat, and deletes it when the
// check ends; a key not renewed within the bucket's age is a builder that died, and is gone. A builder that takes a
// waiting request for a head and trunk commit a key names drops it. **Written by the build seat's holders, several by
// design** — each the key of the check it runs.
CheckRunningBucket = BucketName(ControllerSeat, "checks-running")
)
// The controller's record mirrors (novox/hq ADR 0289, design 54, design 57 package F3): a bucket per kind of
@@ -102,6 +109,10 @@ const RolloutSilentMarkerTTL = time.Minute
// takes nothing live.
const RolloutSilentKeptFor = time.Hour
// CheckRunningKeptFor is the age of a running check's key: a few heartbeats (ten seconds each) of a builder, so a builder that
// died is forgotten within two minutes and the check it ran is taken again then.
const CheckRunningKeptFor = 2 * time.Minute
// AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own.
const AskedKeptFor = 30 * 24 * time.Hour
@@ -130,13 +141,13 @@ func IsControllerBucket(bucket string) bool {
return bucket == CallsBucket || bucket == HandActsBucket || bucket == ConditionsBucket ||
bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket ||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket}
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -264,6 +275,21 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err)
}
// **The checks running: a key that ages** (ADR 0406 §3, issue 974). One message a key, each renewal a new one, so the
// age of the bucket is how long a key outlives the builder that stopped renewing it.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: CheckRunningBucket,
Description: "the pull request checks the builders are running now (novox/hq ADR 0406): `<pull request>` holds the " +
"head and trunk commit, the builder and its build; put by the builder when it starts and at each heartbeat, " +
"deleted when the check ends, gone within two minutes of a builder that died; written by the build seat's holders",
History: 1,
TTL: CheckRunningKeptFor,
MaxValueSize: 4 << 10,
MaxBytes: 8 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
}
// **The record mirrors: no age** (design 54 §5). A key is removed when its record is, by a purge whose
// marker lives a day; a key that expired would be a record the bus forgot while the store still holds it.
for _, m := range []struct{ bucket, why string }{
internal/broker/nats.go
+5
View File
@@ -387,6 +387,11 @@ var DegradableGrants = []DegradableGrant{
"first push, and a check asked before that is asked again by the next announcement of its head (ADR 0406 §3, " +
"issue 974)",
},
{
Subject: "$KV." + CheckRunningBucket + ".>",
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
"holders do (ADR 0406 §3, issue 974)",
},
{
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
internal/broker/testdata/composed.conf
Vendored
+1 -1
View File
@@ -24,7 +24,7 @@ accounts {
jetstream: enabled
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
internal/broker/writers.go
+12
View File
@@ -41,6 +41,14 @@ func ownMachine(p Principal, pattern string) bool {
return p.Kind == KindNode && len(tokens) > 2 && tokens[2] == p.Node
}
// holdsTheBuildSeat is a principal holding the build seat, or the controller, which asserts the bucket the holders write.
func holdsTheBuildSeat(p Principal, _ string) bool {
if p.Kind == KindController {
return true
}
return holdsSeatOf(p, "mesh.seat."+TheBuildSeat+".x")
}
// holdsSeatOf is a principal holding the seat a `mesh.seat.<seat>.…` pattern names: its module's own
// principal, or the node tools carrying a module that holds it (ADR 0175, the runtime is its modules).
func holdsSeatOf(p Principal, pattern string) bool {
@@ -153,6 +161,10 @@ var WritersTable = []WriterRow{
Others: "the build seat's holders take it",
Subjects: []string{CheckSubjectPrefix(TheBuildSeat, true) + ">", CheckSubjectPrefix(TheBuildSeat, false) + ">"},
Writes: isController},
{State: "a pull request's check running", Writer: "the build seat's holders — several by design, each the key of the check it runs",
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
internal/broker/writers_test.go
+1
View File
@@ -28,6 +28,7 @@ var designRows = []string{
"stream definitions and bus permissions",
"builds and their outcomes",
"a pull request's check asked",
"a pull request's check running",
"a merge announced",
"a pull request's head announced",
"a pull request's merge check",