Let a build agent be paused, have a build killed, and end an ask cancelled as it took it (hq ADR 0219)
A queued ask could only be waited out and a running build only ended by stopping the machine, which redelivered it elsewhere. The holder now serves current, kill, pause and resume on its own machine's subjects; a kill ends the build's process group and labelled containers and settles the ask as failed, killed by hand; pause is kept in the workspace across a restart and said on the bus. The controller writes cancelled ids to a cancelled set the holder reads on taking an ask, closing the race a delete alone leaves.
This commit is contained in:
jochen
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -131,6 +132,14 @@ type natsMachine struct {
|
||||
on string
|
||||
seat string
|
||||
sub *nats.Subscription
|
||||
opts MachineOptions
|
||||
}
|
||||
|
||||
// MachineOptions is what a holder tells the taking loop about itself (novox/hq ADR 0219).
|
||||
type MachineOptions struct {
|
||||
// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
|
||||
// already running finishes. Nil is never paused.
|
||||
Paused func() bool
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -145,6 +154,14 @@ func MachineOverNATSOn(js *broker.JetStream, on, seat string) BuildMachine {
|
||||
return &natsMachine{js: js, on: on, seat: seat}
|
||||
}
|
||||
|
||||
// MachineOverNATSWith is MachineOverNATSOn for a holder that can be paused (novox/hq ADR 0219).
|
||||
func MachineOverNATSWith(js *broker.JetStream, on, seat string, opts MachineOptions) BuildMachine {
|
||||
return &natsMachine{js: js, on: on, seat: seat, opts: opts}
|
||||
}
|
||||
|
||||
// pausedPoll is how often a paused holder looks again whether it was resumed.
|
||||
const pausedPoll = 2 * time.Second
|
||||
|
||||
func (m *natsMachine) Close() {
|
||||
if m.sub != nil {
|
||||
_ = m.sub.Unsubscribe()
|
||||
@@ -189,6 +206,17 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
// **Paused takes nothing new** (novox/hq ADR 0219). Asked before the fetch, never during a
|
||||
// build: what this machine already took it finishes, and what it has not taken stays in the
|
||||
// queue for another holder — or for this one, resumed.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pausedPoll):
|
||||
}
|
||||
continue
|
||||
}
|
||||
// One, and wait a while for it; an empty queue is a timeout, which is the normal state of a
|
||||
// machine with nothing to build, and is asked again.
|
||||
fetched, err := sub.Fetch(1, nats.Context(ctx))
|
||||
@@ -220,12 +248,25 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working)
|
||||
do(ctx, &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat})
|
||||
do(ctx, build)
|
||||
close(working)
|
||||
}
|
||||
}
|
||||
@@ -307,3 +348,17 @@ func (b *natsBuild) Say(step, message string) {
|
||||
}
|
||||
|
||||
func (b *natsBuild) Hold(after time.Duration) error { return b.msg.NakWithDelay(after) }
|
||||
|
||||
// endCancelled settles an ask that was cancelled as it was taken: its outcome said as failed, as the
|
||||
// controller already recorded it, so anybody still waiting on it hears the answer — then
|
||||
// terminated, so the queue neither keeps nor redelivers it.
|
||||
func endCancelled(ctx context.Context, b *natsBuild) {
|
||||
r := b.request
|
||||
fmt.Fprintf(os.Stderr, "%s was cancelled by hand as this machine took it; not built\n", r.ID)
|
||||
result := BuildResult{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: b.on,
|
||||
Source: r.Source, DryRun: r.DryRun, Failed: CancelledByHand}
|
||||
if err := b.Announce(ctx, result); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say %s was cancelled: %v\n", r.ID, err)
|
||||
}
|
||||
_ = b.msg.Term()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The build queue, read and changed by hand (novox/hq ADR 0219).
|
||||
//
|
||||
// **A queue nobody can see is a queue nobody can trust.** Until this, the build seat's work queue
|
||||
// was visible only as its consequences: a plan LATE with nothing to say why, a build that never
|
||||
// started because five asks ahead of it were waiting on a laptop that was shut. ADR 0219 makes the
|
||||
// queue the controller's to show and change, and the build running on one machine that machine's
|
||||
// holder's to end.
|
||||
//
|
||||
// Three kinds of ask are in the seat's stream at any moment, told apart by the worker consumer
|
||||
// every holder pulls from:
|
||||
//
|
||||
// - **waiting** — after the last one the worker handed out (stream seq > delivered.stream_seq);
|
||||
// - **in flight** — handed out and not yet settled, which the holder that took it said with its
|
||||
// `started` event, and which the worker counts among its acks pending;
|
||||
// - **dead** — handed out as many times as the worker allows and never settled. A work queue
|
||||
// drops what it settles, so one still in the stream behind the worker is either in flight or
|
||||
// dead; the started events and the worker's pending count say which.
|
||||
//
|
||||
// Everything that drops an ask leaves a failed outcome for it, recorded the way a failed build's is
|
||||
// (`cancelled by hand`, `killed by hand`), so a plan waiting on it fails visibly rather than waiting
|
||||
// for ever on an ask nobody will answer.
|
||||
|
||||
// The words an outcome says when a person ended the ask.
|
||||
const (
|
||||
CancelledByHand = "cancelled by hand"
|
||||
KilledByHand = "killed by hand"
|
||||
)
|
||||
|
||||
// Ask states in a queue.
|
||||
const (
|
||||
AskWaiting = "waiting"
|
||||
AskInFlight = "in flight"
|
||||
AskDead = "dead"
|
||||
)
|
||||
|
||||
// QueuedAsk is one ask in the seat's work queue.
|
||||
type QueuedAsk struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Request BuildRequest `json:"-"`
|
||||
ID string `json:"id"`
|
||||
// Repository, Path and Ref are the ask's, as the request carried them; Held never travels out of
|
||||
// here — it is every artifact the mesh has built, and a queue listing is not where that belongs.
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
AskedAt time.Time `json:"asked-at,omitempty"`
|
||||
State string `json:"state"`
|
||||
// On and Started are the holder that took an in-flight ask and when, from its started event.
|
||||
On string `json:"on,omitempty"`
|
||||
Started time.Time `json:"started,omitempty"`
|
||||
}
|
||||
|
||||
// Queue is the seat's work queue as it stands.
|
||||
type Queue struct {
|
||||
Seat string `json:"seat"`
|
||||
// Delivered is the last stream sequence the worker handed out; AckPending how many it handed out
|
||||
// and has not had settled; MaxDeliver how many times it hands one ask out.
|
||||
Delivered uint64 `json:"delivered"`
|
||||
AckPending int `json:"ack-pending"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []QueuedAsk `json:"asks"`
|
||||
}
|
||||
|
||||
// Of is the asks in one state, in queue order.
|
||||
func (q Queue) Of(state string) []QueuedAsk {
|
||||
var out []QueuedAsk
|
||||
for _, a := range q.Asks {
|
||||
if a.State == state {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Find is the ask with this id.
|
||||
func (q Queue) Find(id string) (QueuedAsk, bool) {
|
||||
for _, a := range q.Asks {
|
||||
if a.ID == id {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return QueuedAsk{}, false
|
||||
}
|
||||
|
||||
// BuildHeard is what the seat's events say about builds: the latest start of each id, and every id
|
||||
// whose outcome was heard.
|
||||
type BuildHeard struct {
|
||||
Started map[string]BuildStart
|
||||
Outcomes map[string]bool
|
||||
}
|
||||
|
||||
// ClassifyQueue says which state each ask is in. Pure: the stream's asks in sequence order, what the
|
||||
// worker says, and what the seat's events said.
|
||||
//
|
||||
// **In flight is what a machine is building now.** A holder builds one ask at a time (ADR 0190), so
|
||||
// what a machine is building is its latest start, if no outcome has been heard for it. An ask
|
||||
// behind the worker that is some machine's current build is in flight; at most AckPending of them
|
||||
// are, newest start first — a machine that died mid-build has a latest start forever, and the
|
||||
// worker's own count is what bounds that. An ask behind the worker that no machine has said it
|
||||
// started is in flight only while the worker counts more pending than starts explain (taken, and
|
||||
// not yet said); otherwise it is dead.
|
||||
//
|
||||
// **The worker's count is the server's, and it lags in one case**: an ask handed out its last time
|
||||
// and handed back is still counted pending until some holder pulls again, when the server finds it
|
||||
// past its deliveries and lets it go. Holders pull whenever they are idle, so this is a moment —
|
||||
// except while every holder is paused, when such an ask reads as in flight with no machine named.
|
||||
// Cancel takes an ask in flight that no machine said it started, for exactly that reason.
|
||||
func ClassifyQueue(asks []QueuedAsk, delivered uint64, ackPending int, heard BuildHeard) []QueuedAsk {
|
||||
// Each machine's latest start.
|
||||
latest := map[string]BuildStart{}
|
||||
for _, s := range heard.Started {
|
||||
at := startedAt(s)
|
||||
if was, ok := latest[s.On]; !ok || at.After(startedAt(was)) {
|
||||
latest[s.On] = s
|
||||
}
|
||||
}
|
||||
current := map[string]BuildStart{}
|
||||
for _, s := range latest {
|
||||
if !heard.Outcomes[s.ID] {
|
||||
current[s.ID] = s
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]QueuedAsk, len(asks))
|
||||
copy(out, asks)
|
||||
var running, unsaid []int
|
||||
for i := range out {
|
||||
a := &out[i]
|
||||
if a.Seq > delivered {
|
||||
a.State = AskWaiting
|
||||
continue
|
||||
}
|
||||
a.State = AskDead
|
||||
if s, ok := current[a.ID]; ok {
|
||||
a.On, a.Started = s.On, startedAt(s)
|
||||
running = append(running, i)
|
||||
continue
|
||||
}
|
||||
if _, ever := heard.Started[a.ID]; !ever {
|
||||
unsaid = append(unsaid, i)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(running, func(x, y int) bool { return out[running[x]].Started.After(out[running[y]].Started) })
|
||||
slots := ackPending
|
||||
for _, i := range running {
|
||||
if slots == 0 {
|
||||
// A machine's latest start the worker no longer counts: it died with the ask, and the
|
||||
// ask was handed out as often as it may be.
|
||||
out[i].On, out[i].Started = "", time.Time{}
|
||||
continue
|
||||
}
|
||||
out[i].State = AskInFlight
|
||||
slots--
|
||||
}
|
||||
for _, i := range unsaid {
|
||||
if slots == 0 {
|
||||
break
|
||||
}
|
||||
out[i].State = AskInFlight
|
||||
slots--
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func startedAt(s BuildStart) time.Time {
|
||||
t, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||
return t
|
||||
}
|
||||
|
||||
// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the
|
||||
// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO,
|
||||
// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches.
|
||||
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
q := Queue{Seat: seat}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
stream, err := api.Stream(ctx, worker.Stream)
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
|
||||
}
|
||||
consumer, err := stream.Consumer(ctx, worker.Name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrConsumerNotFound):
|
||||
// No holder has ever been given a worker: everything in the stream is waiting.
|
||||
case err != nil:
|
||||
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
default:
|
||||
info, err := consumer.Info(ctx)
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
|
||||
}
|
||||
q.Delivered = info.Delivered.Stream
|
||||
q.AckPending = info.NumAckPending
|
||||
q.MaxDeliver = info.Config.MaxDeliver
|
||||
}
|
||||
|
||||
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
|
||||
// is a handful of reads, never the history.
|
||||
var asks []QueuedAsk
|
||||
var seq uint64 = 1
|
||||
for n := 0; n < 10000; n++ {
|
||||
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
|
||||
}
|
||||
ask := QueuedAsk{Seq: msg.Sequence}
|
||||
if json.Unmarshal(msg.Data, &ask.Request) == nil {
|
||||
r := ask.Request
|
||||
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
ask.AskedAt = at
|
||||
}
|
||||
}
|
||||
asks = append(asks, ask)
|
||||
seq = msg.Sequence + 1
|
||||
}
|
||||
|
||||
// What the events say, from shortly before the oldest ask behind the worker: its start, if any,
|
||||
// came after it was asked.
|
||||
var since time.Time
|
||||
for _, a := range asks {
|
||||
if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) {
|
||||
since = a.AskedAt
|
||||
}
|
||||
}
|
||||
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
|
||||
if len(asks) > 0 && q.Delivered > 0 {
|
||||
if since.IsZero() {
|
||||
since = time.Now().Add(-7 * 24 * time.Hour)
|
||||
}
|
||||
if heard, err = ReadBuildEvents(ctx, js, seat, since.Add(-time.Minute)); err != nil {
|
||||
return q, err
|
||||
}
|
||||
}
|
||||
q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard)
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// ReadBuildEvents is every start and outcome the seat announced since a moment, from the events
|
||||
// stream, with a consumer of its own that is gone when this returns.
|
||||
func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, since time.Time) (BuildHeard, error) {
|
||||
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
|
||||
// One token after `event`: started and built, never a build's log lines, which are two.
|
||||
subject := "mesh.seat." + seat + ".event.*"
|
||||
sub, err := js.Context().PullSubscribe(subject, "",
|
||||
nats.BindStream(broker.EventsStream), nats.StartTime(since), nats.AckNone())
|
||||
if err != nil {
|
||||
return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for {
|
||||
batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return heard, fmt.Errorf("reading %s: %w", subject, err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
switch msg.Subject {
|
||||
case BuildStartedOf(seat):
|
||||
var s BuildStart
|
||||
if json.Unmarshal(msg.Data, &s) == nil && s.ID != "" {
|
||||
if was, ok := heard.Started[s.ID]; !ok || startedAt(s).After(startedAt(was)) {
|
||||
heard.Started[s.ID] = s
|
||||
}
|
||||
}
|
||||
case BuildOutcomeOf(seat):
|
||||
var r BuildResult
|
||||
if json.Unmarshal(msg.Data, &r) == nil && r.ID != "" {
|
||||
heard.Outcomes[r.ID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(batch) < 500 {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return heard, ctx.Err()
|
||||
}
|
||||
}
|
||||
return heard, nil
|
||||
}
|
||||
|
||||
// --- the cancelled set ----------------------------------------------------------------------
|
||||
|
||||
// safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern.
|
||||
var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
// Cancelled is what the controller writes for an ask it cancelled.
|
||||
type Cancelled struct {
|
||||
At string `json:"at"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// MarkCancelled puts an ask's id into the seat's cancelled set (novox/hq ADR 0219). The controller's
|
||||
// act, before it deletes the ask from the queue.
|
||||
func MarkCancelled(ctx context.Context, js *broker.JetStream, seat, id string) error {
|
||||
if !safeKey.MatchString(id) {
|
||||
return fmt.Errorf("%q is not a build id", id)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.CancelledSetName(seat))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the cancelled set of %s is not on the bus — the controller asserts it at its "+
|
||||
"start, so one older than this has not: %w", seat, err)
|
||||
}
|
||||
body, err := json.Marshal(Cancelled{At: time.Now().UTC().Format(time.RFC3339Nano), Why: CancelledByHand})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, id, body)
|
||||
return err
|
||||
}
|
||||
|
||||
// IsCancelled asks the seat's cancelled set whether an ask was cancelled: one direct read of one key,
|
||||
// which is all a holder is granted of it.
|
||||
func IsCancelled(conn *nats.Conn, seat, id string) (bool, error) {
|
||||
if !safeKey.MatchString(id) {
|
||||
// Not a key the controller could have written.
|
||||
return false, nil
|
||||
}
|
||||
set := broker.CancelledSetName(seat)
|
||||
reply, err := conn.Request("$JS.API.DIRECT.GET.KV_"+set+".$KV."+set+"."+id, nil, 3*time.Second)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return cancelledFrom(reply)
|
||||
}
|
||||
|
||||
// cancelledFrom reads a direct get's answer: a value is a cancel; not found, or a deletion marker,
|
||||
// is not.
|
||||
func cancelledFrom(reply *nats.Msg) (bool, error) {
|
||||
if reply.Header != nil {
|
||||
switch reply.Header.Get("Status") {
|
||||
case "":
|
||||
case "404":
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the cancelled set answered %s %s",
|
||||
reply.Header.Get("Status"), reply.Header.Get("Description"))
|
||||
}
|
||||
if op := reply.Header.Get("KV-Operation"); op == "DEL" || op == "PURGE" {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return len(reply.Data) > 0, nil
|
||||
}
|
||||
|
||||
// --- a holder's verbs -----------------------------------------------------------------------
|
||||
|
||||
// NodeSeatToolSubject is where a node-scoped seat's verb is asked of one machine's holder (design 33
|
||||
// §4): the seat's verb subject with the machine as its last token.
|
||||
func NodeSeatToolSubject(seat, verb, node string) string {
|
||||
return SeatToolSubject(seat, verb) + "." + node
|
||||
}
|
||||
|
||||
// AskSeatTool asks one machine's holder of a node seat one of its verbs and reads its answer.
|
||||
func AskSeatTool(ctx context.Context, conn *nats.Conn, seat, verb, node string, args any,
|
||||
timeout time.Duration) (Answer, error) {
|
||||
body, err := json.Marshal(args)
|
||||
if err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
asking, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
reply, err := conn.RequestWithContext(asking, NodeSeatToolSubject(seat, verb, node), body)
|
||||
switch {
|
||||
case errors.Is(err, nats.ErrNoResponders):
|
||||
return Answer{}, fmt.Errorf("nothing on %s answers %s.%s: its holder is not running, or is "+
|
||||
"older than the verb", node, seat, verb)
|
||||
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout):
|
||||
return Answer{}, fmt.Errorf("%s did not answer %s.%s within %s", node, seat, verb, timeout)
|
||||
case err != nil:
|
||||
return Answer{}, err
|
||||
}
|
||||
var answer Answer
|
||||
if err := json.Unmarshal(reply.Data, &answer); err != nil {
|
||||
return Answer{}, fmt.Errorf("%s answered %s.%s with something unreadable: %w", node, seat, verb, err)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// --- a holder saying whether it takes work ----------------------------------------------------
|
||||
|
||||
// HolderState is what a holder says about itself whenever it is paused or resumed, at its start,
|
||||
// and while it stays paused: whether it takes work (novox/hq ADR 0219). Retained on the events
|
||||
// stream under the machine's own subject, so the last one is the machine's state.
|
||||
type HolderState struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// BuildPausedOf is where one machine's holder of a build seat says whether it takes work.
|
||||
func BuildPausedOf(seat, node string) string { return "mesh.seat." + seat + ".event.paused." + node }
|
||||
|
||||
// PausedSaid reads what each machine last said about taking work. A machine that never said is not
|
||||
// in the answer — a holder older than ADR 0219 takes work and never pauses.
|
||||
func PausedSaid(js *broker.JetStream, seat string, nodes []string) (map[string]HolderState, error) {
|
||||
out := map[string]HolderState{}
|
||||
for _, node := range nodes {
|
||||
msg, err := js.Context().GetLastMsg(broker.EventsStream, BuildPausedOf(seat, node))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var s HolderState
|
||||
if json.Unmarshal(msg.Data, &s) == nil {
|
||||
out[node] = s
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The build queue read and changed by hand (novox/hq ADR 0219).
|
||||
|
||||
// Which ask is waiting, in flight and dead, from the stream, the worker and the seat's events.
|
||||
func TestTheQueueTellsWaitingInFlightAndDeadApart(t *testing.T) {
|
||||
at := func(m int) string {
|
||||
return time.Date(2026, 10, 5, 12, m, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
}
|
||||
asks := []QueuedAsk{
|
||||
{Seq: 1, ID: "dead-1"}, // handed out five times, its machine moved on: dead
|
||||
{Seq: 2, ID: "running-g"}, // g14's latest start, no outcome: in flight
|
||||
{Seq: 3, ID: "running-a"}, // ace's latest start, no outcome: in flight
|
||||
{Seq: 4, ID: "unsaid"}, // taken, not yet said: in flight while the worker counts it
|
||||
{Seq: 5, ID: "waiting-1"}, // after the worker's last delivery
|
||||
{Seq: 6, ID: "waiting-2"},
|
||||
}
|
||||
heard := BuildHeard{
|
||||
Started: map[string]BuildStart{
|
||||
"dead-1": {ID: "dead-1", On: "g14", At: at(1)},
|
||||
"running-g": {ID: "running-g", On: "g14", At: at(5)},
|
||||
"running-a": {ID: "running-a", On: "ace", At: at(6)},
|
||||
"done": {ID: "done", On: "novox", At: at(7)},
|
||||
},
|
||||
Outcomes: map[string]bool{"done": true},
|
||||
}
|
||||
got := ClassifyQueue(asks, 4, 3, heard)
|
||||
want := map[string]string{"dead-1": AskDead, "running-g": AskInFlight, "running-a": AskInFlight,
|
||||
"unsaid": AskInFlight, "waiting-1": AskWaiting, "waiting-2": AskWaiting}
|
||||
for _, a := range got {
|
||||
if a.State != want[a.ID] {
|
||||
t.Errorf("%s is %s, want %s", a.ID, a.State, want[a.ID])
|
||||
}
|
||||
}
|
||||
q := Queue{Asks: got}
|
||||
if a, _ := q.Find("running-a"); a.On != "ace" || a.Started.IsZero() {
|
||||
t.Errorf("an ask in flight does not say where: %+v", a)
|
||||
}
|
||||
|
||||
// **The worker's count bounds it**: with two pending, the machine whose start is oldest died with
|
||||
// its ask, and the ask nobody said is not in flight either.
|
||||
got = ClassifyQueue(asks, 4, 2, heard)
|
||||
q = Queue{Asks: got}
|
||||
if a, _ := q.Find("running-g"); a.State != AskInFlight {
|
||||
t.Errorf("running-g is %s", a.State)
|
||||
}
|
||||
if a, _ := q.Find("unsaid"); a.State != AskDead {
|
||||
t.Errorf("an ask no start explains, past the worker's count, is %s", a.State)
|
||||
}
|
||||
// None pending: everything behind the worker is dead, and names no machine.
|
||||
for _, a := range ClassifyQueue(asks, 4, 0, heard) {
|
||||
if a.Seq <= 4 && (a.State != AskDead || a.On != "") {
|
||||
t.Errorf("%s with nothing pending is %s on %q", a.ID, a.State, a.On)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What a direct read of the cancelled set answers.
|
||||
func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) {
|
||||
found := &nats.Msg{Header: nats.Header{"Nats-Subject": []string{"$KV.x.build-1"}}, Data: []byte(`{"why":"cancelled by hand"}`)}
|
||||
if c, err := cancelledFrom(found); err != nil || !c {
|
||||
t.Errorf("a value read as %v %v", c, err)
|
||||
}
|
||||
missing := &nats.Msg{Header: nats.Header{"Status": []string{"404"}}}
|
||||
if c, err := cancelledFrom(missing); err != nil || c {
|
||||
t.Errorf("not found read as %v %v", c, err)
|
||||
}
|
||||
deleted := &nats.Msg{Header: nats.Header{"KV-Operation": []string{"DEL"}}}
|
||||
if c, err := cancelledFrom(deleted); err != nil || c {
|
||||
t.Errorf("a deletion marker read as %v %v", c, err)
|
||||
}
|
||||
broken := &nats.Msg{Header: nats.Header{"Status": []string{"408"}, "Description": []string{"Request Timeout"}}}
|
||||
if _, err := cancelledFrom(broken); err == nil {
|
||||
t.Error("an error answer read as an answer")
|
||||
}
|
||||
if c, err := IsCancelled(nil, TheBuildMachine, "a.b.>"); err != nil || c {
|
||||
t.Error("a pattern was looked up as a key")
|
||||
}
|
||||
}
|
||||
|
||||
// --- against a real server ----------------------------------------------------------------------
|
||||
|
||||
func aBusWithACancelledSet(t *testing.T) *broker.JetStream {
|
||||
t.Helper()
|
||||
js := aBusWithTheBuildRole(t)
|
||||
seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}
|
||||
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) })
|
||||
return js
|
||||
}
|
||||
|
||||
// **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by
|
||||
// the holder that took it — terminated, never built, its outcome said as failed.
|
||||
func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) {
|
||||
js := aBusWithACancelledSet(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = outcomes.Unsubscribe() }()
|
||||
_ = js.Conn().Flush()
|
||||
for _, id := range []string{"build-cancelled", "build-kept"} {
|
||||
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
built := make(chan string, 2)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
built <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case id := <-built:
|
||||
if id != "build-kept" {
|
||||
t.Fatalf("%s was built", id)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the ask that was not cancelled was never built")
|
||||
}
|
||||
msg, err := outcomes.NextMsg(5 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said BuildResult
|
||||
if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" ||
|
||||
said.Failed != CancelledByHand || said.On != "anchor" {
|
||||
t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err)
|
||||
}
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the cancelled ask is still queued: terminated, it would not be")
|
||||
}
|
||||
|
||||
// A paused holder takes nothing; resumed, it takes what waited.
|
||||
func TestNatsAPausedHolderTakesNothingNew(t *testing.T) {
|
||||
js := aBusWithACancelledSet(t)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
paused := make(chan bool, 1)
|
||||
paused <- true
|
||||
isPaused := func() bool {
|
||||
p := <-paused
|
||||
paused <- p
|
||||
return p
|
||||
}
|
||||
took := make(chan string, 1)
|
||||
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused})
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"})
|
||||
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case id := <-took:
|
||||
t.Fatalf("a paused holder took %s", id)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
q, err := ReadQueue(ctx, js, TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting {
|
||||
t.Fatalf("while paused the queue reads %+v", q.Asks)
|
||||
}
|
||||
<-paused
|
||||
paused <- false
|
||||
select {
|
||||
case id := <-took:
|
||||
if id != "build-waits" {
|
||||
t.Fatalf("took %s", id)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("resumed, the holder never took what waited")
|
||||
}
|
||||
}
|
||||
|
||||
// What a holder last said about taking work is read back per machine.
|
||||
func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} {
|
||||
body, _ := json.Marshal(s)
|
||||
if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 {
|
||||
t.Fatalf("read back %+v", said)
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,18 @@ const RebindAfter = 30 * time.Second
|
||||
// tried again (2026-09-30). A refused subscription is therefore retried until it holds: the server
|
||||
// says so asynchronously and invalidates the subscription, which is what is checked.
|
||||
func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) {
|
||||
return b.serveTools(seat, func(verb string) string { return SeatToolSubject(seat, verb) }, handlers, logger)
|
||||
}
|
||||
|
||||
// ServeNodeSeatTools is ServeSeatTools for one machine's holder of a node-scoped seat (novox/hq ADR
|
||||
// 0159, ADR 0219): each verb on the seat's subject for this machine and no other, so a call names
|
||||
// the machine it is for and only that machine's holder answers it.
|
||||
func (b OverNATS) ServeNodeSeatTools(seat, node string, handlers map[string]ToolHandler, logger *log.Logger) (func(), error) {
|
||||
return b.serveTools(seat, func(verb string) string { return NodeSeatToolSubject(seat, verb, node) }, handlers, logger)
|
||||
}
|
||||
|
||||
func (b OverNATS) serveTools(seat string, subjectOf func(string) string, handlers map[string]ToolHandler,
|
||||
logger *log.Logger) (func(), error) {
|
||||
var subs []*nats.Subscription
|
||||
done := make(chan struct{})
|
||||
stop := func() {
|
||||
@@ -52,7 +64,7 @@ func (b OverNATS) ServeSeatTools(seat string, handlers map[string]ToolHandler, l
|
||||
}
|
||||
for verb, handle := range handlers {
|
||||
verb, handle := verb, handle
|
||||
subject := SeatToolSubject(seat, verb)
|
||||
subject := subjectOf(verb)
|
||||
bind := func() (*nats.Subscription, error) {
|
||||
return b.Conn.QueueSubscribe(subject, "seat."+seat, func(msg *nats.Msg) {
|
||||
// Its own goroutine per call: a slow `push` must not hold up a `status` asked beside it,
|
||||
|
||||
Reference in New Issue
Block a user