Files
mesh-delivery/modules/mesh-delivery-pipeline/internal/pipeline/queue.go
T
jschoubben bb930de377 R12: the queue's approval wait, stall and cap
A pipeline whose every step not ended is an approval asked of the operator leaves the bound: its
running key goes and the next waiting is placed; answered and still running, it is keyed running
at once, over the bound. After every placing, pipelines waiting with none running raise the finding
pipelines-stalled naming the first waiting, cleared once one runs. A start beyond
pipelines-waiting-cap is an ordinary waiting pipeline and raises queue-full naming it, cleared when
an end brings the waiting back to the cap. Both findings are the scope delivery's, the kind as id,
warnings with no owner.

Builds: design 47 H §748; lead answers H-A-14, H-A-36, H-A-52, H-A-106

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 04:56:31 +01:00

446 lines
16 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// ErrNoKey is a QueueStore's answer to a Get of a key it does not hold.
var ErrNoKey = errors.New("the queue holds no such key")
// QueueStore is the runner's state queue (design 47 H §748, H-A-36, H-A-72 (2), H-A-96): its counter queue.next, one key
// waiting.<sequence> per waiting pipeline and one key running.<pipeline> per pipeline whose steps are asked. A write is by
// compare-and-set: Create on a key's absence (ErrExists when it is written), Update on the revision read (ErrChanged).
type QueueStore interface {
Get(key string) ([]byte, uint64, error)
Create(key string, value []byte) (uint64, error)
Update(key string, value []byte, revision uint64) (uint64, error)
Delete(key string) error
// Keys are the keys beginning with prefix, in key order.
Keys(prefix string) ([]string, error)
}
// Queue bounds the pipelines whose steps are asked at once to AtOnce, the setting pipelines-at-once (design 47 H §748,
// H-A-14): it is judged against the running.<pipeline> keys alone (H-A-96); a pipeline over it waits under the next value
// of queue.next.
type Queue struct {
Store QueueStore
AtOnce int
// Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap.
Cap int
// Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes
// none.
Findings Findings
Observer string
Now func() time.Time
}
const (
runningPrefix = "running."
waitingPrefix = "waiting."
)
// sequenceKey is a waiting pipeline's key, its sequence 20 digits, so the lowest is read first by key order (H-A-72 (2)).
func sequenceKey(seq int64) string {
return seat.QueueByWaitingSequenceKey(fmt.Sprintf("%020d", seq))
}
// Running are the pipelines whose running.<pipeline> key is written.
func (q *Queue) Running() ([]string, error) {
ks, err := q.Store.Keys(runningPrefix)
if err != nil {
return nil, fmt.Errorf("the queue's running pipelines are not read: %w", err)
}
out := make([]string, 0, len(ks))
for _, k := range ks {
if p, ok := seat.ParseQueueByRunningPipelineKey(k); ok {
out = append(out, p)
}
}
return out, nil
}
// Waiting are the waiting.<sequence> keys and the pipeline each names, lowest sequence first.
func (q *Queue) Waiting() ([]string, []string, error) {
ks, err := q.Store.Keys(waitingPrefix)
if err != nil {
return nil, nil, fmt.Errorf("the queue's waiting pipelines are not read: %w", err)
}
slices.Sort(ks)
var keys, pipelines []string
for _, k := range ks {
raw, _, err := q.Store.Get(k)
if errors.Is(err, ErrNoKey) {
continue
} else if err != nil {
return nil, nil, fmt.Errorf("the queue's %s is not read: %w", k, err)
}
w, err := seat.DecodeQueueByWaitingSequenceValue(raw)
if err != nil {
return nil, nil, fmt.Errorf("the queue's %s is no waiting pipeline: %w", k, err)
}
keys, pipelines = append(keys, k), append(pipelines, w.Pipeline)
}
return keys, pipelines, nil
}
// full says whether as many pipelines run as AtOnce allows.
func (q *Queue) full() (bool, error) {
running, err := q.Running()
if err != nil {
return false, err
}
return len(running) >= max(q.AtOnce, 1), nil
}
// run writes key's running.<pipeline> by compare-and-set on its absence; it says whether it wrote it.
func (q *Queue) run(key string) (bool, error) {
raw, err := json.Marshal(seat.QueueByRunningPipelineValue{Pipeline: key})
if err != nil {
return false, err
}
if _, err := q.Store.Create(seat.QueueByRunningPipelineKey(key), raw); errors.Is(err, ErrExists) {
return false, nil
} else if err != nil {
return false, fmt.Errorf("the queue's running key of %s is not written: %w", key, err)
}
return true, nil
}
// next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1.
func (q *Queue) next() (int64, error) {
for {
raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey())
var seq int64 = 1
if err == nil {
if seq, err = seat.DecodeQueueByQueueNextValue(raw); err != nil {
return 0, fmt.Errorf("the queue's counter is no number: %w", err)
}
} else if !errors.Is(err, ErrNoKey) {
return 0, fmt.Errorf("the queue's counter is not read: %w", err)
}
value := []byte(strconv.FormatInt(seq+1, 10))
if errors.Is(err, ErrNoKey) {
_, err = q.Store.Create(seat.QueueByQueueNextKey(), value)
} else {
_, err = q.Store.Update(seat.QueueByQueueNextKey(), value, rev)
}
if errors.Is(err, ErrExists) || errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return 0, fmt.Errorf("the queue's counter is not written: %w", err)
}
return seq, nil
}
}
// wait writes key's waiting.<sequence>.
func (q *Queue) wait(seq int64, key string) error {
raw, err := json.Marshal(seat.QueueByWaitingSequenceValue{Pipeline: key})
if err != nil {
return err
}
if _, err := q.Store.Create(sequenceKey(seq), raw); err != nil {
return fmt.Errorf("the queue's waiting key of %s is not written: %w", key, err)
}
return nil
}
// leave deletes key's running.<pipeline>, a key not written being no error.
func (q *Queue) leave(key string) error {
if err := q.Store.Delete(seat.QueueByRunningPipelineKey(key)); err != nil && !errors.Is(err, ErrNoKey) {
return fmt.Errorf("the queue's running key of %s is not deleted: %w", key, err)
}
return nil
}
// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on <branch>.<source> with step pipeline and
// on pipelines-at-once (design 47 H §748, H-A-52).
func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait {
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run,
Step: seat.WaitStepPipeline, On: seat.WaitOnPipelinesAtOnce, How: how, Since: at}
}
func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how string) {
if s.Say == nil {
return
}
if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil {
s.logf("%s of %s is not said: %v", event, p.Pipeline, err)
}
}
// queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key
// written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key
// is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A
// start with no queue is placed.
func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) {
if s.Queue == nil {
return true, "", nil
}
full, err := s.Queue.full()
if err != nil || !full {
return err == nil, "", err
}
seq, err := s.Queue.next()
if err != nil {
return false, "", err
}
p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting
return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline)
}
// ran writes the running key of a pipeline placed at its start, after the pipeline is written: one written but not keyed
// (the runner stopped between) runs once over the bound, never holding it.
func (s *Starter) ran(key string) {
if s.Queue == nil {
return
}
if _, err := s.Queue.run(key); err != nil {
s.logf("%s runs, and %v", key, err)
}
}
// StartQueue takes up the queue at the runner's start, from its own state (§748, H-A-14): a running key whose pipeline no
// longer runs (its end heard while the key was not deleted) is deleted, then the waiting are placed (PlaceQueued).
func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
running, err := s.Queue.Running()
if err != nil {
return 0, err
}
for _, key := range running {
if p, _, err := s.Store.Get(key); err == nil && p.Outcome == seat.PipelineOutcomeRunning {
continue
}
s.logf("%s is keyed running in the queue and runs no more: its key is deleted", key)
if err := s.Queue.leave(key); err != nil {
return 0, err
}
}
return s.PlaceQueued(view)
}
// Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the
// queue placed (PlaceQueued).
func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
if err := s.Queue.leave(key); err != nil {
return 0, err
}
return s.PlaceQueued(view)
}
// PlaceQueued places the waiting pipelines, lowest sequence first, while fewer than pipelines-at-once run (design 47 H
// §748): at the runner's start, from its own state, and on every end. Each placed has its running key written and its
// waiting key deleted, becomes running, has its steps placed as a started pipeline's are, and says wait-ended. A waiting key
// whose pipeline is no longer waiting is deleted. It answers how many it placed.
func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
defer s.Queue.judge(s.logf)
placedN := 0
for {
full, err := s.Queue.full()
if err != nil || full {
return placedN, err
}
keys, pipelines, err := s.Queue.Waiting()
if err != nil || len(keys) == 0 {
return placedN, err
}
ok, err := s.placeQueued(keys[0], pipelines[0], view)
if err != nil {
return placedN, err
}
if ok {
placedN++
}
}
}
// placeQueued places the waiting pipeline key, its queue key wkey; false when it was no longer waiting (its key dropped).
func (s *Starter) placeQueued(wkey, key string, view func(branch string) (Half, bool)) (bool, error) {
for {
p, rev, err := s.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting {
if err != nil {
s.logf("the queue's %s names %s, not read (%v): its key is dropped", wkey, key, err)
}
return false, s.dropWaiting(wkey)
}
if _, err := s.Queue.run(key); err != nil {
return false, err
}
now := s.now()
p.Outcome = seat.PipelineOutcomeRunning
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written running: %w", key, err)
}
if err := s.dropWaiting(wkey); err != nil {
return true, err
}
s.logf("%s is placed: its turn under pipelines-at-once came", key)
s.sayQueueWait(seat.EventWaitEnded, p, now, "ran")
if p.Run != "" {
// A run's steps are placed from its candidate's newest revision, as at the runner's start (A-222).
if s.RunTurn != nil {
s.RunTurn(UnplacedRun{Run: p.Run, Branch: p.Branch})
}
return true, nil
}
if s.Registration != nil && view != nil {
if _, err := s.placeWaiting(s.Store, func(x seat.Pipeline) bool { return x.Pipeline == key }, s.Registration, view, "its turn in the queue"); err != nil {
return true, err
}
}
return true, nil
}
}
func (s *Starter) dropWaiting(wkey string) error {
if err := s.Queue.Store.Delete(wkey); err != nil && !errors.Is(err, ErrNoKey) {
return fmt.Errorf("the queue's %s is not deleted: %w", wkey, err)
}
return nil
}
// Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the
// bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)).
func (s *Starter) Rejoin(key string) {
if s.Queue == nil {
return
}
if ok, err := s.Queue.run(key); err != nil {
s.logf("%s runs again after its approval, and %v", key, err)
} else if ok {
s.logf("%s runs again after its approval: keyed running at once", key)
}
s.Queue.judge(s.logf)
}
// awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a
// pipeline does not count against pipelines-at-once (§748).
func awaitsApprovalOnly(p seat.Pipeline) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
asked := false
for _, st := range p.Steps {
if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" {
return false
}
asked = true
}
return asked
}
// The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id.
const (
PipelinesStalledKind = "pipelines-stalled"
QueueFullKind = "queue-full"
)
// waitingCount is how many pipelines wait, -1 when the queue is not read.
func (q *Queue) waitingCount() int {
ks, err := q.Store.Keys(waitingPrefix)
if err != nil {
return -1
}
return len(ks)
}
// overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)).
func (q *Queue) overCap(pipeline string, logf func(string, ...any)) {
if q.Cap <= 0 {
return
}
if n := q.waitingCount(); n > q.Cap {
q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline),
"More pipelines wait than the queue's cap",
fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce),
"fewer pipelines waiting than pipelines-waiting-cap", logf)
}
}
// judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or
// none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106).
func (q *Queue) judge(logf func(string, ...any)) {
if q.Findings == nil {
return
}
running, err := q.Running()
if err != nil {
logf("the queue is not judged: %v", err)
return
}
_, waiting, err := q.Waiting()
if err != nil {
logf("the queue is not judged: %v", err)
return
}
if len(waiting) > 0 && len(running) == 0 {
q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]),
"Pipelines wait and none runs",
fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]),
"a pipeline running", logf)
} else {
q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf)
}
if q.Cap <= 0 || len(waiting) <= q.Cap {
q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf)
}
}
func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) {
if q.Findings == nil {
return
}
now := time.Now().UTC()
if q.Now != nil {
now = q.Now().UTC()
}
err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind,
Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs,
Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}})
if err != nil {
logf("the finding %s is not written: %v", kind, err)
}
}
func (q *Queue) clear(kind, why string, logf func(string, ...any)) {
if q.Findings == nil {
return
}
held := false
for _, f := range q.Findings.Held() {
held = held || (f.Kind == kind && f.ID == kind)
}
if !held {
return
}
if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil {
logf("the finding %s is not cleared: %v", kind, err)
}
}