Hold the build seat's checks while a delivery builds, and let one check that waited 30 minutes through

ADR 0483: the controller keeps a key per plan in its build phase; holders read it and
take no new check while one is there, but for the oldest check past 30 minutes, one
at a time by a let-through key. The queue says why checks wait.
This commit is contained in:
2026-10-21 19:32:47 +02:00
parent 5ba2ffe8e6
commit a14dc04b7b
20 changed files with 1856 additions and 73 deletions
+9
View File
@@ -46,6 +46,7 @@ func CheckStreams() []Stream {
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Direct: true,
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
"request still waiting",
@@ -56,6 +57,7 @@ func CheckStreams() []Stream {
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Direct: true,
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
},
@@ -98,6 +100,13 @@ func holderCheckGrants() []string {
"$KV."+CheckRunningBucket+".>",
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
pub = append(pub, readGrants(PlansBuildingBucket, "")...)
for _, w := range CheckWorkers() {
pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
}
return pub
}
+34
View File
@@ -40,3 +40,37 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
}
}
}
// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
stream := "KV_" + PlansBuildingBucket
for _, s := range []string{
"$JS.API.STREAM.INFO." + stream,
"$JS.API.DIRECT.GET." + stream + ".>",
"$JS.API.CONSUMER.CREATE." + stream + ".>",
"$JS.API.DIRECT.GET.CHECKS_READY",
"$JS.API.DIRECT.GET.CHECKS_OTHER",
} {
has(t, perms.Publish, s)
}
for _, s := range perms.Publish {
if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
}
}
// And the controller alone writes it, as the writers table says.
for _, row := range WritersTable {
for _, s := range row.Subjects {
if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
t.Errorf("%q lets the build seat's holder write the plans building", row.State)
}
}
}
}
+38 -2
View File
@@ -63,6 +63,26 @@ var (
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
// their values to decide that (`link.BuildingPlans`).
PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
)
// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
// (`mesh-controller.plans-building`).
const PlansBuilding = "plans-building"
// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
const (
PlansBuildingKeptFor = 2 * time.Minute
PlansBuildingRefresh = 30 * time.Second
)
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
@@ -204,14 +224,14 @@ func IsControllerBucket(bucket string) bool {
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
bucket == delivery.HaveBucket
bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -354,6 +374,22 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
}
// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
// age of the bucket is how long a key outlives a controller that stopped refreshing it.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: PlansBuildingBucket,
Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
"take no new check while a key is there; written by the controller alone",
History: 1,
TTL: PlansBuildingKeptFor,
MaxValueSize: 4 << 10,
MaxBytes: 8 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
}
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: DeliveryMergesBucket,
+5
View File
@@ -412,6 +412,11 @@ var DegradableGrants = []DegradableGrant{
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
"holders do (ADR 0406 §3, issue 974)",
},
{
Subject: "$KV." + PlansBuildingBucket + ".>",
Reason: "refused: the controller then says no plan builds, and no check is held while a delivery builds — a check run " +
"wrongly costs a delivery a place, a check held wrongly costs a pull request its turn (ADR 0483 clause 2, issue 1687)",
},
{
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
+134
View File
@@ -0,0 +1,134 @@
//go:build integration
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// **A holder of the build seat reads the plans building and the server refuses its write** (novox/hq ADR 0483 clause 2, issue
// 1687): on a real server whose users are composed as the controller composes them, the controller makes the bucket and puts a
// plan's key; the build agent's credential lists the keys and gets one, and its put of a key is refused with a permissions
// violation. It also reads the first waiting ask of a check queue by a direct get, which takes nothing from the queue.
func TestNatsTheBuildSeatsHoldersReadThePlansBuildingAndTheServerRefusesTheirWrite(t *testing.T) { // 1687
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
"alpha": {{Module: RuntimeModule}, {Module: "build-agent", Holds: []Seat{seat}}},
}})
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(s.Shutdown)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
connect := func(user string) (*nats.Conn, chan error) {
t.Helper()
refused := make(chan error, 8)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
if err != nil {
t.Fatalf("%s cannot connect: %v", user, err)
}
t.Cleanup(nc.Close)
return nc, refused
}
controller, _ := connect(ControllerName)
cj := OnConn(controller)
if err := AssertMeshStreams(cj); err != nil {
t.Fatalf("the controller cannot make the streams: %v", err)
}
if err := cj.EnsureControllerBuckets(); err != nil {
t.Fatalf("the controller cannot make its buckets: %v", err)
}
cjs, _ := jetstream.New(controller)
ckv, err := cjs.KeyValue(ctx, PlansBuildingBucket)
if err != nil {
t.Fatal(err)
}
if _, err := ckv.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
t.Fatalf("the controller cannot put a plan's key: %v", err)
}
holder, refused := connect("alpha.build-agent")
hjs, err := jetstream.New(holder)
if err != nil {
t.Fatal(err)
}
kv, err := hjs.KeyValue(ctx, PlansBuildingBucket)
if err != nil {
t.Fatalf("a holder of the build seat cannot bind %s: %v", PlansBuildingBucket, err)
}
keys, err := kv.Keys(ctx)
if err != nil || len(keys) != 1 || keys[0] != "plan-1" {
t.Fatalf("the holder lists %v (%v), want plan-1", keys, err)
}
if e, err := kv.Get(ctx, "plan-1"); err != nil || e == nil {
t.Fatalf("the holder cannot get the key: %v", err)
}
// Its write is refused by the server.
if err := holder.Publish("$KV."+PlansBuildingBucket+".plan-2", []byte(`{}`)); err != nil {
t.Fatal(err)
}
_ = holder.Flush()
select {
case err := <-refused:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("the server said %v, want a permissions violation", err)
}
case <-time.After(3 * time.Second):
t.Error("the server did not refuse the holder's put of a plan's key")
}
if _, err := ckv.Get(ctx, "plan-2"); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Errorf("the refused put left a key: %v", err)
}
// And the direct get of a check queue's first waiting ask, which is granted.
reply, err := holder.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+ChecksOtherStream, []byte(`{"seq":1,"next_by_subj":"`+
CheckSubjectPrefix(TheBuildSeat, false)+`>"}`))
if err != nil {
t.Fatalf("the holder's direct get of %s was not answered: %v", ChecksOtherStream, err)
}
if reply.Header.Get("Status") != "404" {
t.Errorf("an empty queue answered %q, want 404", reply.Header.Get("Status"))
}
}
+1 -1
View File
@@ -29,7 +29,7 @@ accounts {
]
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
+4
View File
@@ -177,6 +177,10 @@ var WritersTable = []WriterRow{
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
{State: "a delivery plan in its build phase", Writer: "controller (lease holder), at the first build's ask and at each refresh",
KeptIn: "key-value " + PlansBuildingBucket + ", aged out when not refreshed",
Others: "the build seat's holders read it and take no new check while a key is there (ADR 0483)",
Subjects: kvOf(PlansBuildingBucket), Writes: isController},
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
+2
View File
@@ -29,6 +29,8 @@ var designRows = []string{
"builds and their outcomes",
"a pull request's check asked",
"a pull request's check running",
// The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687).
"a delivery plan in its build phase",
"a merge announced",
"a pull request's head announced",
"a pull request's merge check",