package pipeline import ( "context" "encoding/json" "errors" "fmt" "slices" "strconv" "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) // ErrNoKey is a QueueStore's answer to a Get of a key it does not hold. var ErrNoKey = errors.New("the queue holds no such key") // QueueStore is the runner's state queue (design 47 H §748, H-A-36, H-A-72 (2), H-A-96): its counter queue.next, one key // waiting. per waiting pipeline and one key running. per pipeline whose steps are asked. A write is by // compare-and-set: Create on a key's absence (ErrExists when it is written), Update on the revision read (ErrChanged). type QueueStore interface { Get(key string) ([]byte, uint64, error) Create(key string, value []byte) (uint64, error) Update(key string, value []byte, revision uint64) (uint64, error) Delete(key string) error // Keys are the keys beginning with prefix, in key order. Keys(prefix string) ([]string, error) } // Queue bounds the pipelines whose steps are asked at once to AtOnce, the setting pipelines-at-once (design 47 H §748, // H-A-14): it is judged against the running. keys alone (H-A-96); a pipeline over it waits under the next value // of queue.next. type Queue struct { Store QueueStore AtOnce int // Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap. Cap int // Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes // none. Findings Findings Observer string Now func() time.Time } const ( runningPrefix = "running." waitingPrefix = "waiting." ) // sequenceKey is a waiting pipeline's key, its sequence 20 digits, so the lowest is read first by key order (H-A-72 (2)). func sequenceKey(seq int64) string { return seat.QueueByWaitingSequenceKey(fmt.Sprintf("%020d", seq)) } // Running are the pipelines whose running. key is written. func (q *Queue) Running() ([]string, error) { ks, err := q.Store.Keys(runningPrefix) if err != nil { return nil, fmt.Errorf("the queue's running pipelines are not read: %w", err) } out := make([]string, 0, len(ks)) for _, k := range ks { if p, ok := seat.ParseQueueByRunningPipelineKey(k); ok { out = append(out, p) } } return out, nil } // Waiting are the waiting. keys and the pipeline each names, lowest sequence first. func (q *Queue) Waiting() ([]string, []string, error) { ks, err := q.Store.Keys(waitingPrefix) if err != nil { return nil, nil, fmt.Errorf("the queue's waiting pipelines are not read: %w", err) } slices.Sort(ks) var keys, pipelines []string for _, k := range ks { raw, _, err := q.Store.Get(k) if errors.Is(err, ErrNoKey) { continue } else if err != nil { return nil, nil, fmt.Errorf("the queue's %s is not read: %w", k, err) } w, err := seat.DecodeQueueByWaitingSequenceValue(raw) if err != nil { return nil, nil, fmt.Errorf("the queue's %s is no waiting pipeline: %w", k, err) } keys, pipelines = append(keys, k), append(pipelines, w.Pipeline) } return keys, pipelines, nil } // full says whether as many pipelines run as AtOnce allows. func (q *Queue) full() (bool, error) { running, err := q.Running() if err != nil { return false, err } return len(running) >= max(q.AtOnce, 1), nil } // run writes key's running. by compare-and-set on its absence; it says whether it wrote it. func (q *Queue) run(key string) (bool, error) { raw, err := json.Marshal(seat.QueueByRunningPipelineValue{Pipeline: key}) if err != nil { return false, err } if _, err := q.Store.Create(seat.QueueByRunningPipelineKey(key), raw); errors.Is(err, ErrExists) { return false, nil } else if err != nil { return false, fmt.Errorf("the queue's running key of %s is not written: %w", key, err) } return true, nil } // next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1. func (q *Queue) next() (int64, error) { for { raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey()) var seq int64 = 1 if err == nil { if seq, err = seat.DecodeQueueByQueueNextValue(raw); err != nil { return 0, fmt.Errorf("the queue's counter is no number: %w", err) } } else if !errors.Is(err, ErrNoKey) { return 0, fmt.Errorf("the queue's counter is not read: %w", err) } value := []byte(strconv.FormatInt(seq+1, 10)) if errors.Is(err, ErrNoKey) { _, err = q.Store.Create(seat.QueueByQueueNextKey(), value) } else { _, err = q.Store.Update(seat.QueueByQueueNextKey(), value, rev) } if errors.Is(err, ErrExists) || errors.Is(err, ErrChanged) { continue } else if err != nil { return 0, fmt.Errorf("the queue's counter is not written: %w", err) } return seq, nil } } // wait writes key's waiting.. func (q *Queue) wait(seq int64, key string) error { raw, err := json.Marshal(seat.QueueByWaitingSequenceValue{Pipeline: key}) if err != nil { return err } if _, err := q.Store.Create(sequenceKey(seq), raw); err != nil { return fmt.Errorf("the queue's waiting key of %s is not written: %w", key, err) } return nil } // leave deletes key's running., a key not written being no error. func (q *Queue) leave(key string) error { if err := q.Store.Delete(seat.QueueByRunningPipelineKey(key)); err != nil && !errors.Is(err, ErrNoKey) { return fmt.Errorf("the queue's running key of %s is not deleted: %w", key, err) } return nil } // queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and // on pipelines-at-once (design 47 H §748, H-A-52). func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait { return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run, Step: seat.WaitStepPipeline, On: seat.WaitOnPipelinesAtOnce, How: how, Since: at} } func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how string) { if s.Say == nil { return } if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil { s.logf("%s of %s is not said: %v", event, p.Pipeline, err) } } // queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key // written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key // is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A // start with no queue is placed. func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) { if s.Queue == nil { return true, "", nil } full, err := s.Queue.full() if err != nil || !full { return err == nil, "", err } seq, err := s.Queue.next() if err != nil { return false, "", err } p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline) } // ran writes the running key of a pipeline placed at its start, after the pipeline is written: one written but not keyed // (the runner stopped between) runs once over the bound, never holding it. func (s *Starter) ran(key string) { if s.Queue == nil { return } if _, err := s.Queue.run(key); err != nil { s.logf("%s runs, and %v", key, err) } } // StartQueue takes up the queue at the runner's start, from its own state (§748, H-A-14): a running key whose pipeline no // longer runs (its end heard while the key was not deleted) is deleted, then the waiting are placed (PlaceQueued). func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) { if s.Queue == nil { return 0, nil } running, err := s.Queue.Running() if err != nil { return 0, err } for _, key := range running { if p, _, err := s.Store.Get(key); err == nil && p.Outcome == seat.PipelineOutcomeRunning { continue } s.logf("%s is keyed running in the queue and runs no more: its key is deleted", key) if err := s.Queue.leave(key); err != nil { return 0, err } } return s.PlaceQueued(view) } // Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the // queue placed (PlaceQueued). func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) { if s.Queue == nil { return 0, nil } if err := s.Queue.leave(key); err != nil { return 0, err } return s.PlaceQueued(view) } // PlaceQueued places the waiting pipelines, lowest sequence first, while fewer than pipelines-at-once run (design 47 H // §748): at the runner's start, from its own state, and on every end. Each placed has its running key written and its // waiting key deleted, becomes running, has its steps placed as a started pipeline's are, and says wait-ended. A waiting key // whose pipeline is no longer waiting is deleted. It answers how many it placed. func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error) { if s.Queue == nil { return 0, nil } defer s.Queue.judge(s.logf) placedN := 0 for { full, err := s.Queue.full() if err != nil || full { return placedN, err } keys, pipelines, err := s.Queue.Waiting() if err != nil || len(keys) == 0 { return placedN, err } ok, err := s.placeQueued(keys[0], pipelines[0], view) if err != nil { return placedN, err } if ok { placedN++ } } } // placeQueued places the waiting pipeline key, its queue key wkey; false when it was no longer waiting (its key dropped). func (s *Starter) placeQueued(wkey, key string, view func(branch string) (Half, bool)) (bool, error) { for { p, rev, err := s.Store.Get(key) if err != nil || p.Outcome != seat.PipelineOutcomeWaiting { if err != nil { s.logf("the queue's %s names %s, not read (%v): its key is dropped", wkey, key, err) } return false, s.dropWaiting(wkey) } if _, err := s.Queue.run(key); err != nil { return false, err } now := s.now() p.Outcome = seat.PipelineOutcomeRunning if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { continue } else if err != nil { return false, fmt.Errorf("the pipeline %s is not written running: %w", key, err) } if err := s.dropWaiting(wkey); err != nil { return true, err } s.logf("%s is placed: its turn under pipelines-at-once came", key) s.sayQueueWait(seat.EventWaitEnded, p, now, "ran") if p.Run != "" { // A run's steps are placed from its candidate's newest revision, as at the runner's start (A-222). if s.RunTurn != nil { s.RunTurn(UnplacedRun{Run: p.Run, Branch: p.Branch}) } return true, nil } if s.Registration != nil && view != nil { if _, err := s.placeWaiting(s.Store, func(x seat.Pipeline) bool { return x.Pipeline == key }, s.Registration, view, "its turn in the queue"); err != nil { return true, err } } return true, nil } } func (s *Starter) dropWaiting(wkey string) error { if err := s.Queue.Store.Delete(wkey); err != nil && !errors.Is(err, ErrNoKey) { return fmt.Errorf("the queue's %s is not deleted: %w", wkey, err) } return nil } // Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the // bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)). func (s *Starter) Rejoin(key string) { if s.Queue == nil { return } if ok, err := s.Queue.run(key); err != nil { s.logf("%s runs again after its approval, and %v", key, err) } else if ok { s.logf("%s runs again after its approval: keyed running at once", key) } s.Queue.judge(s.logf) } // awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a // pipeline does not count against pipelines-at-once (§748). func awaitsApprovalOnly(p seat.Pipeline) bool { if p.Outcome != seat.PipelineOutcomeRunning { return false } asked := false for _, st := range p.Steps { if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting { continue } if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" { return false } asked = true } return asked } // The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id. const ( PipelinesStalledKind = "pipelines-stalled" QueueFullKind = "queue-full" ) // waitingCount is how many pipelines wait, -1 when the queue is not read. func (q *Queue) waitingCount() int { ks, err := q.Store.Keys(waitingPrefix) if err != nil { return -1 } return len(ks) } // overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)). func (q *Queue) overCap(pipeline string, logf func(string, ...any)) { if q.Cap <= 0 { return } if n := q.waitingCount(); n > q.Cap { q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline), "More pipelines wait than the queue's cap", fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce), "fewer pipelines waiting than pipelines-waiting-cap", logf) } } // judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or // none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106). func (q *Queue) judge(logf func(string, ...any)) { if q.Findings == nil { return } running, err := q.Running() if err != nil { logf("the queue is not judged: %v", err) return } _, waiting, err := q.Waiting() if err != nil { logf("the queue is not judged: %v", err) return } if len(waiting) > 0 && len(running) == 0 { q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]), "Pipelines wait and none runs", fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]), "a pipeline running", logf) } else { q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf) } if q.Cap <= 0 || len(waiting) <= q.Cap { q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf) } } func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) { if q.Findings == nil { return } now := time.Now().UTC() if q.Now != nil { now = q.Now().UTC() } err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind, Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs, Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}}) if err != nil { logf("the finding %s is not written: %v", kind, err) } } func (q *Queue) clear(kind, why string, logf func(string, ...any)) { if q.Findings == nil { return } held := false for _, f := range q.Findings.Held() { held = held || (f.Kind == kind && f.ID == kind) } if !held { return } if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil { logf("the finding %s is not cleared: %v", kind, err) } }