Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)
A queued ask could only be waited out and a running build only ended by stopping the machine, which redelivered it elsewhere. The holder now serves current, kill, pause and resume on its own machine's subjects; a kill ends the build's process group and labelled containers and settles the ask as failed, killed by hand; pause is kept in the workspace across a restart and said on the bus. The controller writes cancelled ids to a cancelled set the holder reads on taking an ask, closing the race a delete alone leaves.
This commit is contained in:
jochen
@@ -0,0 +1,92 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// A seat's cancelled set (novox/hq ADR 0219).
|
||||
//
|
||||
// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
|
||||
// race no ordering on one side closes: a holder may fetch the ask in the moment between the
|
||||
// controller reading the queue and deleting the message, and build what a person cancelled. So the
|
||||
// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
|
||||
// there after taking it and before building: listed, it terminates the ask and announces it failed
|
||||
// rather than starting it.
|
||||
//
|
||||
// **A key-value bucket, because that is the shape the bus already has for "a small set the
|
||||
// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
|
||||
// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
|
||||
// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
|
||||
// three objects of one work queue read as one family; asserted by the controller with the queue,
|
||||
// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
|
||||
|
||||
// CancelledSetName is the bucket holding a seat's cancelled asks.
|
||||
func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
|
||||
|
||||
// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
|
||||
// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
|
||||
func hasCancelledSet(seat string) bool {
|
||||
for _, s := range seatsTheControllerAsks {
|
||||
if s == seat {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
|
||||
// own, and never one to report as state nothing declares.
|
||||
func IsCancelledSet(bucket string) bool {
|
||||
return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
|
||||
}
|
||||
|
||||
// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
|
||||
const cancelledSetAge = 7 * 24 * time.Hour
|
||||
|
||||
// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
|
||||
type CancelledSetAsserter interface {
|
||||
EnsureCancelledSet(seat string) error
|
||||
}
|
||||
|
||||
// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
|
||||
func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
|
||||
for _, s := range seats {
|
||||
if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
|
||||
continue
|
||||
}
|
||||
if err := a.EnsureCancelledSet(s.Name); err != nil {
|
||||
return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
|
||||
// Direct reads on, which is how a holder looks an id up with one request.
|
||||
func (j *JetStream) EnsureCancelledSet(seat string) error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CancelledSetName(seat),
|
||||
Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
|
||||
"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
|
||||
"so an ask fetched in that moment is ended rather than built", seat),
|
||||
History: 1,
|
||||
TTL: cancelledSetAge,
|
||||
MaxValueSize: 4 * 1024,
|
||||
MaxBytes: 4 * 1024 * 1024,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package broker
|
||||
|
||||
import "testing"
|
||||
|
||||
// A build agent reads its seat's cancelled set by key and nothing more, answers its verbs on its own
|
||||
// machine's subjects, and says whether it is paused under its own machine's name; the controller may
|
||||
// ask any machine's holder its verbs (novox/hq ADR 0219).
|
||||
func TestTheBuildQueueIsControlledWithTheGrantsItNeedsAndNoMore(t *testing.T) {
|
||||
seat := Seat{Name: "node-build-agent", Scope: "node", Accepts: []string{"build"},
|
||||
Emits: []string{"started", "built", "log.*", "paused.*"},
|
||||
Serves: []string{"current", "kill", "pause", "resume"}}
|
||||
holder, err := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "build-agent",
|
||||
Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
has(t, holder.Publish, "$JS.API.DIRECT.GET.KV_SEAT_NODE_BUILD_AGENT_cancelled.$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||
hasNot(t, holder.Publish, "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>")
|
||||
has(t, holder.Publish, "mesh.seat.node-build-agent.event.paused.*")
|
||||
has(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.ace")
|
||||
hasNot(t, holder.Subscribe, "mesh.seat.node-build-agent.tool.kill.g14")
|
||||
|
||||
// A module's own seat's queue is its own affair: no cancelled set, no grant for one.
|
||||
other, _ := PermissionsFor(Principal{Kind: KindModule, Node: "ace", Module: "telegram",
|
||||
Holds: []Seat{{Name: "telegram-sender", Accepts: []string{"send"}}}, PasswordHash: "x"})
|
||||
hasNot(t, other.Publish, "$JS.API.DIRECT.GET.KV_SEAT_TELEGRAM_SENDER_cancelled.$KV.SEAT_TELEGRAM_SENDER_cancelled.>")
|
||||
|
||||
controller, _ := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
has(t, controller.Publish, "mesh.seat.node-build-agent.tool.>")
|
||||
|
||||
if CancelledSetName("node-build-agent") != "SEAT_NODE_BUILD_AGENT_cancelled" ||
|
||||
!IsCancelledSet("SEAT_NODE_BUILD_AGENT_cancelled") || IsCancelledSet("build-agent_cancelled") {
|
||||
t.Error("the cancelled set is not named as its seat's family")
|
||||
}
|
||||
}
|
||||
|
||||
// Only the work queues the controller asks get a cancelled set.
|
||||
func TestOnlyTheControllersQueuesHaveACancelledSet(t *testing.T) {
|
||||
var asserted []string
|
||||
a := asserterFunc(func(seat string) error { asserted = append(asserted, seat); return nil })
|
||||
if err := RaiseCancelledSets(a, []DeclaredSeat{
|
||||
{Name: "node-build-agent", Accepts: []string{"build"}},
|
||||
{Name: "telegram-sender", Accepts: []string{"send"}},
|
||||
{Name: "mesh-controller"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asserted) != 1 || asserted[0] != "node-build-agent" {
|
||||
t.Fatalf("asserted %v", asserted)
|
||||
}
|
||||
}
|
||||
|
||||
type asserterFunc func(string) error
|
||||
|
||||
func (f asserterFunc) EnsureCancelledSet(seat string) error { return f(seat) }
|
||||
|
||||
// A seat's cancelled set is never reported as state nothing declares.
|
||||
func TestACancelledSetIsNotUndeclaredState(t *testing.T) {
|
||||
undeclared, err := RaiseBuckets(fakeBuckets{names: []string{"SEAT_NODE_BUILD_AGENT_cancelled", "gone_state"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(undeclared) != 1 || undeclared[0] != "gone_state" {
|
||||
t.Fatalf("undeclared %v", undeclared)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeBuckets struct{ names []string }
|
||||
|
||||
func (f fakeBuckets) EnsureBucket(Bucket) error { return nil }
|
||||
func (f fakeBuckets) BucketNames() ([]string, error) { return f.names, nil }
|
||||
@@ -221,6 +221,10 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// the role, and whichever machine holding it is idle takes it.
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
pub = append(pub, "mesh.seat."+seat+".accept.>")
|
||||
// **And its holders' verbs, on every machine** (novox/hq ADR 0219): what a holder is
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||
@@ -404,6 +408,15 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
"$JS.API.CONSUMER.INFO."+stream+"."+worker,
|
||||
"$JS.API.CONSUMER.MSG.NEXT."+stream+"."+worker,
|
||||
"$JS.ACK."+stream+"."+worker+".>")
|
||||
// **And whether an ask it took was cancelled** (novox/hq ADR 0219): one key of the
|
||||
// seat's cancelled set, read directly by its id, so a holder that fetched an ask in the
|
||||
// moment the controller cancelled it ends it instead of building it. Read, never written:
|
||||
// the set is the controller's, and only the work queues the controller asks — and so may
|
||||
// cancel from — have one.
|
||||
if hasCancelledSet(s.Name) {
|
||||
set := CancelledSetName(s.Name)
|
||||
pub = append(pub, "$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+".>")
|
||||
}
|
||||
for _, a := range s.Accepts {
|
||||
sub = append(sub, seatSubject(s, "accept", a))
|
||||
}
|
||||
|
||||
@@ -160,7 +160,8 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
if !declared[n] {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
||||
if !declared[n] && !IsCancelledSet(n) {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
Reference in New Issue
Block a user