novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

issue 974: the checks are queued by pull request on two work-queue streams, taken after a delivery build, dropped when judged, stopped when their key moves; the reordering and the forge-asking end are gone

Browse Source
This commit is contained in:
jschoubben
2026-10-17 14:01:33 +02:00
parent f883830975
commit 3d09d5235f
20 changed files with 888 additions and 1412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/link/queue.go
+99 -49
View File
@@ -62,11 +62,12 @@ type QueuedAsk struct {
Path string `json:"path,omitempty"`
Ref string `json:"ref,omitempty"`
AskedAt time.Time `json:"asked-at,omitempty"`
// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
// Stored is when the stream took it.
Stored time.Time `json:"stored,omitempty"`
State string `json:"state"`
// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
movedFrom string
// Stream is the work queue holding it: the seat's own, or one of the two check queues (novox/hq ADR 0406 §3). Its
// Seq is the sequence there.
Stream string `json:"stream,omitempty"`
// On and Started are the holder building an in-flight ask and when it started, from its started
// event. Was is the holder that started an in-flight ask and is no longer building it — handed
// back, to be handed out again — with Started its start there.
@@ -201,62 +202,113 @@ func startedAt(s BuildStart) time.Time {
return t
}
// ReadQueue reads a build seat's work queue: the stream's asks, the worker's position, and what the
// seat's events said about the builds behind it. Through the JetStream API alone (STREAM.INFO,
// CONSUMER.INFO, STREAM.MSG.GET), which only the controller's account reaches.
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
// queueStream is one stream a seat's asks wait in, and the worker that hands them out.
type queueStream struct {
Worker broker.Consumer
Filter string
}
// queueStreamsOf are the streams a seat's asks wait in, in the order a builder takes from them (novox/hq ADR 0406 §3): the
// seat's own queue, where a delivery build waits, then the checks of pull requests marked ready, then every other check.
// Only the current build role has the check queues; the retired one's checks wait on its own.
func queueStreamsOf(seat string) []queueStream {
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
streams := []queueStream{{Worker: worker, Filter: BuildWorkOf(seat)}}
if seat == broker.TheBuildSeat {
for _, w := range broker.CheckWorkers() {
streams = append(streams, queueStream{Worker: w, Filter: w.Filters[0]})
}
}
return streams
}
// WorkerOf is the worker that hands out an ask, by the queue it waits in.
func WorkerOf(seat string, ask QueuedAsk) broker.Consumer {
streams := queueStreamsOf(seat)
for _, qs := range streams {
if qs.Worker.Stream == ask.Stream {
return qs.Worker
}
}
return streams[0].Worker
}
// ReadQueue reads a build seat's work queues: the streams' asks, the workers' positions, and what the seat's events
// said about the builds behind them. Through the JetStream API alone (STREAM.INFO, CONSUMER.INFO, STREAM.MSG.GET),
// which only the controller's account reaches. The asks come in the order they are handed out: the seat's own queue's,
// the ready checks', the other checks'.
func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, error) {
q := Queue{Seat: seat}
api, err := jetstream.New(js.Conn())
if err != nil {
return q, err
}
stream, err := api.Stream(ctx, worker.Stream)
if err != nil {
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
type read struct {
asks []QueuedAsk
delivered uint64
ackPending int
}
consumer, err := stream.Consumer(ctx, worker.Name)
switch {
case errors.Is(err, jetstream.ErrConsumerNotFound):
// No holder has ever been given a worker: everything in the stream is waiting.
case err != nil:
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
default:
info, err := consumer.Info(ctx)
var reads []read
for i, qs := range queueStreamsOf(seat) {
stream, err := api.Stream(ctx, qs.Worker.Stream)
if err != nil {
return q, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, qs.Worker.Stream, err)
}
var r read
consumer, err := stream.Consumer(ctx, qs.Worker.Name)
switch {
case errors.Is(err, jetstream.ErrConsumerNotFound):
// No holder has ever been given a worker: everything in the stream is waiting.
case err != nil:
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
default:
info, err := consumer.Info(ctx)
if err != nil {
return q, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
}
r.delivered, r.ackPending = info.Delivered.Stream, info.NumAckPending
if i == 0 {
q.Delivered, q.MaxDeliver = info.Delivered.Stream, info.Config.MaxDeliver
}
q.AckPending += info.NumAckPending
if q.MaxDeliver == 0 {
q.MaxDeliver = info.Config.MaxDeliver
}
}
q.Delivered = info.Delivered.Stream
q.AckPending = info.NumAckPending
q.MaxDeliver = info.Config.MaxDeliver
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
// is a handful of reads, never the history.
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(qs.Filter))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
a := askOf(msg)
a.Stream = qs.Worker.Stream
r.asks = append(r.asks, a)
seq = msg.Sequence + 1
}
reads = append(reads, r)
}
// Every ask, by next-by-subject from the first: the stream holds only what is unsettled, so this
// is a handful of reads, never the history.
var asks []QueuedAsk
var seq uint64 = 1
for n := 0; n < 10000; n++ {
msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
if errors.Is(err, jetstream.ErrMsgNotFound) {
break
}
if err != nil {
return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
}
asks = append(asks, askOf(msg))
seq = msg.Sequence + 1
}
// What the events say, from shortly before the oldest ask behind the worker: its start, if any,
// came after it was asked.
// What the events say, from shortly before the oldest ask behind a worker: its start, if any, came after it was asked.
var since time.Time
for _, a := range asks {
if a.Seq <= q.Delivered && (since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since))) {
since = a.AskedAt
behind := false
for _, r := range reads {
for _, a := range r.asks {
if a.Seq <= r.delivered {
behind = true
if since.IsZero() || (!a.AskedAt.IsZero() && a.AskedAt.Before(since)) {
since = a.AskedAt
}
}
}
}
heard := BuildHeard{Started: map[string]BuildStart{}, Outcomes: map[string]bool{}}
if len(asks) > 0 && q.Delivered > 0 {
if behind {
if since.IsZero() {
since = time.Now().Add(-7 * 24 * time.Hour)
}
@@ -264,17 +316,15 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
return q, err
}
}
q.Asks = ClassifyQueue(asks, q.Delivered, q.AckPending, heard)
for _, r := range reads {
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
}
return q, nil
}
// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
ask.Stored = at
}
if json.Unmarshal(msg.Data, &ask.Request) == nil {
r := ask.Request
ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
@@ -282,7 +332,7 @@ func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
ask.AskedAt = at
}
} else {
// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
// Nothing read is nothing kept: a half-read request would pass for a delivery.
ask.Request = BuildRequest{}
}
return ask