A pipeline whose every step not ended is an approval asked of the operator leaves the bound: its running key goes and the next waiting is placed; answered and still running, it is keyed running at once, over the bound. After every placing, pipelines waiting with none running raise the finding pipelines-stalled naming the first waiting, cleared once one runs. A start beyond pipelines-waiting-cap is an ordinary waiting pipeline and raises queue-full naming it, cleared when an end brings the waiting back to the cap. Both findings are the scope delivery's, the kind as id, warnings with no owner. Builds: design 47 H §748; lead answers H-A-14, H-A-36, H-A-52, H-A-106 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
446 lines
16 KiB
Go
446 lines
16 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strconv"
|
|
"time"
|
|
|
|
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
)
|
|
|
|
// ErrNoKey is a QueueStore's answer to a Get of a key it does not hold.
|
|
var ErrNoKey = errors.New("the queue holds no such key")
|
|
|
|
// QueueStore is the runner's state queue (design 47 H §748, H-A-36, H-A-72 (2), H-A-96): its counter queue.next, one key
|
|
// waiting.<sequence> per waiting pipeline and one key running.<pipeline> per pipeline whose steps are asked. A write is by
|
|
// compare-and-set: Create on a key's absence (ErrExists when it is written), Update on the revision read (ErrChanged).
|
|
type QueueStore interface {
|
|
Get(key string) ([]byte, uint64, error)
|
|
Create(key string, value []byte) (uint64, error)
|
|
Update(key string, value []byte, revision uint64) (uint64, error)
|
|
Delete(key string) error
|
|
// Keys are the keys beginning with prefix, in key order.
|
|
Keys(prefix string) ([]string, error)
|
|
}
|
|
|
|
// Queue bounds the pipelines whose steps are asked at once to AtOnce, the setting pipelines-at-once (design 47 H §748,
|
|
// H-A-14): it is judged against the running.<pipeline> keys alone (H-A-96); a pipeline over it waits under the next value
|
|
// of queue.next.
|
|
type Queue struct {
|
|
Store QueueStore
|
|
AtOnce int
|
|
// Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap.
|
|
Cap int
|
|
// Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes
|
|
// none.
|
|
Findings Findings
|
|
Observer string
|
|
Now func() time.Time
|
|
}
|
|
|
|
const (
|
|
runningPrefix = "running."
|
|
waitingPrefix = "waiting."
|
|
)
|
|
|
|
// sequenceKey is a waiting pipeline's key, its sequence 20 digits, so the lowest is read first by key order (H-A-72 (2)).
|
|
func sequenceKey(seq int64) string {
|
|
return seat.QueueByWaitingSequenceKey(fmt.Sprintf("%020d", seq))
|
|
}
|
|
|
|
// Running are the pipelines whose running.<pipeline> key is written.
|
|
func (q *Queue) Running() ([]string, error) {
|
|
ks, err := q.Store.Keys(runningPrefix)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the queue's running pipelines are not read: %w", err)
|
|
}
|
|
out := make([]string, 0, len(ks))
|
|
for _, k := range ks {
|
|
if p, ok := seat.ParseQueueByRunningPipelineKey(k); ok {
|
|
out = append(out, p)
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Waiting are the waiting.<sequence> keys and the pipeline each names, lowest sequence first.
|
|
func (q *Queue) Waiting() ([]string, []string, error) {
|
|
ks, err := q.Store.Keys(waitingPrefix)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("the queue's waiting pipelines are not read: %w", err)
|
|
}
|
|
slices.Sort(ks)
|
|
var keys, pipelines []string
|
|
for _, k := range ks {
|
|
raw, _, err := q.Store.Get(k)
|
|
if errors.Is(err, ErrNoKey) {
|
|
continue
|
|
} else if err != nil {
|
|
return nil, nil, fmt.Errorf("the queue's %s is not read: %w", k, err)
|
|
}
|
|
w, err := seat.DecodeQueueByWaitingSequenceValue(raw)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("the queue's %s is no waiting pipeline: %w", k, err)
|
|
}
|
|
keys, pipelines = append(keys, k), append(pipelines, w.Pipeline)
|
|
}
|
|
return keys, pipelines, nil
|
|
}
|
|
|
|
// full says whether as many pipelines run as AtOnce allows.
|
|
func (q *Queue) full() (bool, error) {
|
|
running, err := q.Running()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return len(running) >= max(q.AtOnce, 1), nil
|
|
}
|
|
|
|
// run writes key's running.<pipeline> by compare-and-set on its absence; it says whether it wrote it.
|
|
func (q *Queue) run(key string) (bool, error) {
|
|
raw, err := json.Marshal(seat.QueueByRunningPipelineValue{Pipeline: key})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := q.Store.Create(seat.QueueByRunningPipelineKey(key), raw); errors.Is(err, ErrExists) {
|
|
return false, nil
|
|
} else if err != nil {
|
|
return false, fmt.Errorf("the queue's running key of %s is not written: %w", key, err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1.
|
|
func (q *Queue) next() (int64, error) {
|
|
for {
|
|
raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey())
|
|
var seq int64 = 1
|
|
if err == nil {
|
|
if seq, err = seat.DecodeQueueByQueueNextValue(raw); err != nil {
|
|
return 0, fmt.Errorf("the queue's counter is no number: %w", err)
|
|
}
|
|
} else if !errors.Is(err, ErrNoKey) {
|
|
return 0, fmt.Errorf("the queue's counter is not read: %w", err)
|
|
}
|
|
value := []byte(strconv.FormatInt(seq+1, 10))
|
|
if errors.Is(err, ErrNoKey) {
|
|
_, err = q.Store.Create(seat.QueueByQueueNextKey(), value)
|
|
} else {
|
|
_, err = q.Store.Update(seat.QueueByQueueNextKey(), value, rev)
|
|
}
|
|
if errors.Is(err, ErrExists) || errors.Is(err, ErrChanged) {
|
|
continue
|
|
} else if err != nil {
|
|
return 0, fmt.Errorf("the queue's counter is not written: %w", err)
|
|
}
|
|
return seq, nil
|
|
}
|
|
}
|
|
|
|
// wait writes key's waiting.<sequence>.
|
|
func (q *Queue) wait(seq int64, key string) error {
|
|
raw, err := json.Marshal(seat.QueueByWaitingSequenceValue{Pipeline: key})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := q.Store.Create(sequenceKey(seq), raw); err != nil {
|
|
return fmt.Errorf("the queue's waiting key of %s is not written: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// leave deletes key's running.<pipeline>, a key not written being no error.
|
|
func (q *Queue) leave(key string) error {
|
|
if err := q.Store.Delete(seat.QueueByRunningPipelineKey(key)); err != nil && !errors.Is(err, ErrNoKey) {
|
|
return fmt.Errorf("the queue's running key of %s is not deleted: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on <branch>.<source> with step pipeline and
|
|
// on pipelines-at-once (design 47 H §748, H-A-52).
|
|
func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait {
|
|
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run,
|
|
Step: seat.WaitStepPipeline, On: seat.WaitOnPipelinesAtOnce, How: how, Since: at}
|
|
}
|
|
|
|
func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how string) {
|
|
if s.Say == nil {
|
|
return
|
|
}
|
|
if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil {
|
|
s.logf("%s of %s is not said: %v", event, p.Pipeline, err)
|
|
}
|
|
}
|
|
|
|
// queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key
|
|
// written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key
|
|
// is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A
|
|
// start with no queue is placed.
|
|
func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) {
|
|
if s.Queue == nil {
|
|
return true, "", nil
|
|
}
|
|
full, err := s.Queue.full()
|
|
if err != nil || !full {
|
|
return err == nil, "", err
|
|
}
|
|
seq, err := s.Queue.next()
|
|
if err != nil {
|
|
return false, "", err
|
|
}
|
|
p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting
|
|
return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline)
|
|
}
|
|
|
|
// ran writes the running key of a pipeline placed at its start, after the pipeline is written: one written but not keyed
|
|
// (the runner stopped between) runs once over the bound, never holding it.
|
|
func (s *Starter) ran(key string) {
|
|
if s.Queue == nil {
|
|
return
|
|
}
|
|
if _, err := s.Queue.run(key); err != nil {
|
|
s.logf("%s runs, and %v", key, err)
|
|
}
|
|
}
|
|
|
|
// StartQueue takes up the queue at the runner's start, from its own state (§748, H-A-14): a running key whose pipeline no
|
|
// longer runs (its end heard while the key was not deleted) is deleted, then the waiting are placed (PlaceQueued).
|
|
func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) {
|
|
if s.Queue == nil {
|
|
return 0, nil
|
|
}
|
|
running, err := s.Queue.Running()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
for _, key := range running {
|
|
if p, _, err := s.Store.Get(key); err == nil && p.Outcome == seat.PipelineOutcomeRunning {
|
|
continue
|
|
}
|
|
s.logf("%s is keyed running in the queue and runs no more: its key is deleted", key)
|
|
if err := s.Queue.leave(key); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
return s.PlaceQueued(view)
|
|
}
|
|
|
|
// Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the
|
|
// queue placed (PlaceQueued).
|
|
func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) {
|
|
if s.Queue == nil {
|
|
return 0, nil
|
|
}
|
|
if err := s.Queue.leave(key); err != nil {
|
|
return 0, err
|
|
}
|
|
return s.PlaceQueued(view)
|
|
}
|
|
|
|
// PlaceQueued places the waiting pipelines, lowest sequence first, while fewer than pipelines-at-once run (design 47 H
|
|
// §748): at the runner's start, from its own state, and on every end. Each placed has its running key written and its
|
|
// waiting key deleted, becomes running, has its steps placed as a started pipeline's are, and says wait-ended. A waiting key
|
|
// whose pipeline is no longer waiting is deleted. It answers how many it placed.
|
|
func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error) {
|
|
if s.Queue == nil {
|
|
return 0, nil
|
|
}
|
|
defer s.Queue.judge(s.logf)
|
|
placedN := 0
|
|
for {
|
|
full, err := s.Queue.full()
|
|
if err != nil || full {
|
|
return placedN, err
|
|
}
|
|
keys, pipelines, err := s.Queue.Waiting()
|
|
if err != nil || len(keys) == 0 {
|
|
return placedN, err
|
|
}
|
|
ok, err := s.placeQueued(keys[0], pipelines[0], view)
|
|
if err != nil {
|
|
return placedN, err
|
|
}
|
|
if ok {
|
|
placedN++
|
|
}
|
|
}
|
|
}
|
|
|
|
// placeQueued places the waiting pipeline key, its queue key wkey; false when it was no longer waiting (its key dropped).
|
|
func (s *Starter) placeQueued(wkey, key string, view func(branch string) (Half, bool)) (bool, error) {
|
|
for {
|
|
p, rev, err := s.Store.Get(key)
|
|
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting {
|
|
if err != nil {
|
|
s.logf("the queue's %s names %s, not read (%v): its key is dropped", wkey, key, err)
|
|
}
|
|
return false, s.dropWaiting(wkey)
|
|
}
|
|
if _, err := s.Queue.run(key); err != nil {
|
|
return false, err
|
|
}
|
|
now := s.now()
|
|
p.Outcome = seat.PipelineOutcomeRunning
|
|
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
|
continue
|
|
} else if err != nil {
|
|
return false, fmt.Errorf("the pipeline %s is not written running: %w", key, err)
|
|
}
|
|
if err := s.dropWaiting(wkey); err != nil {
|
|
return true, err
|
|
}
|
|
s.logf("%s is placed: its turn under pipelines-at-once came", key)
|
|
s.sayQueueWait(seat.EventWaitEnded, p, now, "ran")
|
|
if p.Run != "" {
|
|
// A run's steps are placed from its candidate's newest revision, as at the runner's start (A-222).
|
|
if s.RunTurn != nil {
|
|
s.RunTurn(UnplacedRun{Run: p.Run, Branch: p.Branch})
|
|
}
|
|
return true, nil
|
|
}
|
|
if s.Registration != nil && view != nil {
|
|
if _, err := s.placeWaiting(s.Store, func(x seat.Pipeline) bool { return x.Pipeline == key }, s.Registration, view, "its turn in the queue"); err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
}
|
|
|
|
func (s *Starter) dropWaiting(wkey string) error {
|
|
if err := s.Queue.Store.Delete(wkey); err != nil && !errors.Is(err, ErrNoKey) {
|
|
return fmt.Errorf("the queue's %s is not deleted: %w", wkey, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the
|
|
// bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)).
|
|
func (s *Starter) Rejoin(key string) {
|
|
if s.Queue == nil {
|
|
return
|
|
}
|
|
if ok, err := s.Queue.run(key); err != nil {
|
|
s.logf("%s runs again after its approval, and %v", key, err)
|
|
} else if ok {
|
|
s.logf("%s runs again after its approval: keyed running at once", key)
|
|
}
|
|
s.Queue.judge(s.logf)
|
|
}
|
|
|
|
// awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a
|
|
// pipeline does not count against pipelines-at-once (§748).
|
|
func awaitsApprovalOnly(p seat.Pipeline) bool {
|
|
if p.Outcome != seat.PipelineOutcomeRunning {
|
|
return false
|
|
}
|
|
asked := false
|
|
for _, st := range p.Steps {
|
|
if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
|
continue
|
|
}
|
|
if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" {
|
|
return false
|
|
}
|
|
asked = true
|
|
}
|
|
return asked
|
|
}
|
|
|
|
// The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id.
|
|
const (
|
|
PipelinesStalledKind = "pipelines-stalled"
|
|
QueueFullKind = "queue-full"
|
|
)
|
|
|
|
// waitingCount is how many pipelines wait, -1 when the queue is not read.
|
|
func (q *Queue) waitingCount() int {
|
|
ks, err := q.Store.Keys(waitingPrefix)
|
|
if err != nil {
|
|
return -1
|
|
}
|
|
return len(ks)
|
|
}
|
|
|
|
// overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)).
|
|
func (q *Queue) overCap(pipeline string, logf func(string, ...any)) {
|
|
if q.Cap <= 0 {
|
|
return
|
|
}
|
|
if n := q.waitingCount(); n > q.Cap {
|
|
q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline),
|
|
"More pipelines wait than the queue's cap",
|
|
fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce),
|
|
"fewer pipelines waiting than pipelines-waiting-cap", logf)
|
|
}
|
|
}
|
|
|
|
// judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or
|
|
// none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106).
|
|
func (q *Queue) judge(logf func(string, ...any)) {
|
|
if q.Findings == nil {
|
|
return
|
|
}
|
|
running, err := q.Running()
|
|
if err != nil {
|
|
logf("the queue is not judged: %v", err)
|
|
return
|
|
}
|
|
_, waiting, err := q.Waiting()
|
|
if err != nil {
|
|
logf("the queue is not judged: %v", err)
|
|
return
|
|
}
|
|
if len(waiting) > 0 && len(running) == 0 {
|
|
q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]),
|
|
"Pipelines wait and none runs",
|
|
fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]),
|
|
"a pipeline running", logf)
|
|
} else {
|
|
q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf)
|
|
}
|
|
if q.Cap <= 0 || len(waiting) <= q.Cap {
|
|
q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf)
|
|
}
|
|
}
|
|
|
|
func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) {
|
|
if q.Findings == nil {
|
|
return
|
|
}
|
|
now := time.Now().UTC()
|
|
if q.Now != nil {
|
|
now = q.Now().UTC()
|
|
}
|
|
err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind,
|
|
Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs,
|
|
Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}})
|
|
if err != nil {
|
|
logf("the finding %s is not written: %v", kind, err)
|
|
}
|
|
}
|
|
|
|
func (q *Queue) clear(kind, why string, logf func(string, ...any)) {
|
|
if q.Findings == nil {
|
|
return
|
|
}
|
|
held := false
|
|
for _, f := range q.Findings.Held() {
|
|
held = held || (f.Kind == kind && f.ID == kind)
|
|
}
|
|
if !held {
|
|
return
|
|
}
|
|
if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil {
|
|
logf("the finding %s is not cleared: %v", kind, err)
|
|
}
|
|
}
|