jochen
e610f2d92c
A queued ask could only be waited out and a running build only ended by stopping the machine, which redelivered it elsewhere. The holder now serves current, kill, pause and resume on its own machine's subjects; a kill ends the build's process group and labelled containers and settles the ask as failed, killed by hand; pause is kept in the workspace across a restart and said on the bus. The controller writes cancelled ids to a cancelled set the holder reads on taking an ask, closing the race a delete alone leaves.
93 lines
3.7 KiB
Go
93 lines
3.7 KiB
Go
package broker
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// A seat's cancelled set (novox/hq ADR 0219).
|
|
//
|
|
// **Cancelling an ask is deleting its message from the seat's work queue** — and that alone has a
|
|
// race no ordering on one side closes: a holder may fetch the ask in the moment between the
|
|
// controller reading the queue and deleting the message, and build what a person cancelled. So the
|
|
// controller first writes the ask's id into the seat's cancelled set, and a holder looks its ask up
|
|
// there after taking it and before building: listed, it terminates the ask and announces it failed
|
|
// rather than starting it.
|
|
//
|
|
// **A key-value bucket, because that is the shape the bus already has for "a small set the
|
|
// controller writes and many read cheaply"** (ADR 0201's state buckets): one direct read by key per
|
|
// ask taken — no consumer, no subscription a holder must keep, nothing that grows a holder's grants
|
|
// beyond one read subject. Kept beside the seat's own stream and worker and named like them, so the
|
|
// three objects of one work queue read as one family; asserted by the controller with the queue,
|
|
// and aged out with it — an id cancelled a week ago names an ask the queue no longer holds.
|
|
|
|
// CancelledSetName is the bucket holding a seat's cancelled asks.
|
|
func CancelledSetName(seat string) string { return "SEAT_" + upperSnake(seat) + "_cancelled" }
|
|
|
|
// hasCancelledSet is whether a seat's queue can be cancelled from: the work queues the controller
|
|
// asks, whose asks it alone shows and changes. A module's own seat's queue is that module's affair.
|
|
func hasCancelledSet(seat string) bool {
|
|
for _, s := range seatsTheControllerAsks {
|
|
if s == seat {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// IsCancelledSet says a bucket is a seat's cancelled set rather than a module's state — the mesh's
|
|
// own, and never one to report as state nothing declares.
|
|
func IsCancelledSet(bucket string) bool {
|
|
return strings.HasPrefix(bucket, "SEAT_") && strings.HasSuffix(bucket, "_cancelled")
|
|
}
|
|
|
|
// cancelledSetAge is how long a cancelled id is kept: as long as the seat's queue keeps an ask.
|
|
const cancelledSetAge = 7 * 24 * time.Hour
|
|
|
|
// A CancelledSetAsserter is what raising the cancelled sets needs of a connection.
|
|
type CancelledSetAsserter interface {
|
|
EnsureCancelledSet(seat string) error
|
|
}
|
|
|
|
// RaiseCancelledSets asserts the cancelled set of every work queue the controller asks.
|
|
func RaiseCancelledSets(a CancelledSetAsserter, seats []DeclaredSeat) error {
|
|
for _, s := range seats {
|
|
if len(s.Accepts) == 0 || !hasCancelledSet(s.Name) {
|
|
continue
|
|
}
|
|
if err := a.EnsureCancelledSet(s.Name); err != nil {
|
|
return fmt.Errorf("asserting the cancelled set of %s: %w", s.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// EnsureCancelledSet creates a seat's cancelled set if absent and brings its options to match.
|
|
// Direct reads on, which is how a holder looks an id up with one request.
|
|
func (j *JetStream) EnsureCancelledSet(seat string) error {
|
|
js, err := jetstream.New(j.conn)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
|
Bucket: CancelledSetName(seat),
|
|
Description: fmt.Sprintf("the asks of the %s seat cancelled by hand (novox/hq ADR 0219): written "+
|
|
"by the controller before it deletes an ask from the queue, read by a holder on taking one, "+
|
|
"so an ask fetched in that moment is ended rather than built", seat),
|
|
History: 1,
|
|
TTL: cancelledSetAge,
|
|
MaxValueSize: 4 * 1024,
|
|
MaxBytes: 4 * 1024 * 1024,
|
|
Storage: jetstream.FileStorage,
|
|
}); err != nil {
|
|
return fmt.Errorf("asserting bucket %s: %w", CancelledSetName(seat), err)
|
|
}
|
|
return nil
|
|
}
|