Design 47 H1: the one chain (take-in, forge bridge, pipeline runner, assigner, teller, view, watchman, grants; checker gone) #263
@@ -207,7 +207,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
logger.Printf("the queue: %d pipelines at once, at most %d waiting", atOnce, waitingCap)
|
||||
starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce}
|
||||
starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce, Cap: waitingCap,
|
||||
Findings: rv.Findings, Observer: cred.Module}
|
||||
hearer.Rejoin = starter.Rejoin
|
||||
hearer.Turn = func(key string) {
|
||||
if n, err := starter.Turn(key, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the end of %s placed %d waiting pipelines (%v)", key, n, err)
|
||||
|
||||
@@ -235,6 +235,11 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
// Answered and still running on more than approvals asked, it is keyed running at once, over the bound (§748,
|
||||
// H-A-106 (b)).
|
||||
if !ended && h.Rejoin != nil && p.Outcome == seat.PipelineOutcomeRunning && !awaitsApprovalOnly(p) {
|
||||
h.Rejoin(key)
|
||||
}
|
||||
return p, h.said(p, nil, ended)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -8,6 +9,8 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
@@ -33,6 +36,13 @@ type QueueStore interface {
|
||||
type Queue struct {
|
||||
Store QueueStore
|
||||
AtOnce int
|
||||
// Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap.
|
||||
Cap int
|
||||
// Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes
|
||||
// none.
|
||||
Findings Findings
|
||||
Observer string
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -243,6 +253,7 @@ func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error
|
||||
if s.Queue == nil {
|
||||
return 0, nil
|
||||
}
|
||||
defer s.Queue.judge(s.logf)
|
||||
placedN := 0
|
||||
for {
|
||||
full, err := s.Queue.full()
|
||||
@@ -310,3 +321,125 @@ func (s *Starter) dropWaiting(wkey string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the
|
||||
// bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)).
|
||||
func (s *Starter) Rejoin(key string) {
|
||||
if s.Queue == nil {
|
||||
return
|
||||
}
|
||||
if ok, err := s.Queue.run(key); err != nil {
|
||||
s.logf("%s runs again after its approval, and %v", key, err)
|
||||
} else if ok {
|
||||
s.logf("%s runs again after its approval: keyed running at once", key)
|
||||
}
|
||||
s.Queue.judge(s.logf)
|
||||
}
|
||||
|
||||
// awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a
|
||||
// pipeline does not count against pipelines-at-once (§748).
|
||||
func awaitsApprovalOnly(p seat.Pipeline) bool {
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
return false
|
||||
}
|
||||
asked := false
|
||||
for _, st := range p.Steps {
|
||||
if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting {
|
||||
continue
|
||||
}
|
||||
if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" {
|
||||
return false
|
||||
}
|
||||
asked = true
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id.
|
||||
const (
|
||||
PipelinesStalledKind = "pipelines-stalled"
|
||||
QueueFullKind = "queue-full"
|
||||
)
|
||||
|
||||
// waitingCount is how many pipelines wait, -1 when the queue is not read.
|
||||
func (q *Queue) waitingCount() int {
|
||||
ks, err := q.Store.Keys(waitingPrefix)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return len(ks)
|
||||
}
|
||||
|
||||
// overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)).
|
||||
func (q *Queue) overCap(pipeline string, logf func(string, ...any)) {
|
||||
if q.Cap <= 0 {
|
||||
return
|
||||
}
|
||||
if n := q.waitingCount(); n > q.Cap {
|
||||
q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline),
|
||||
"More pipelines wait than the queue's cap",
|
||||
fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce),
|
||||
"fewer pipelines waiting than pipelines-waiting-cap", logf)
|
||||
}
|
||||
}
|
||||
|
||||
// judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or
|
||||
// none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106).
|
||||
func (q *Queue) judge(logf func(string, ...any)) {
|
||||
if q.Findings == nil {
|
||||
return
|
||||
}
|
||||
running, err := q.Running()
|
||||
if err != nil {
|
||||
logf("the queue is not judged: %v", err)
|
||||
return
|
||||
}
|
||||
_, waiting, err := q.Waiting()
|
||||
if err != nil {
|
||||
logf("the queue is not judged: %v", err)
|
||||
return
|
||||
}
|
||||
if len(waiting) > 0 && len(running) == 0 {
|
||||
q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]),
|
||||
"Pipelines wait and none runs",
|
||||
fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]),
|
||||
"a pipeline running", logf)
|
||||
} else {
|
||||
q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf)
|
||||
}
|
||||
if q.Cap <= 0 || len(waiting) <= q.Cap {
|
||||
q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) {
|
||||
if q.Findings == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if q.Now != nil {
|
||||
now = q.Now().UTC()
|
||||
}
|
||||
err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind,
|
||||
Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs,
|
||||
Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}})
|
||||
if err != nil {
|
||||
logf("the finding %s is not written: %v", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Queue) clear(kind, why string, logf func(string, ...any)) {
|
||||
if q.Findings == nil {
|
||||
return
|
||||
}
|
||||
held := false
|
||||
for _, f := range q.Findings.Held() {
|
||||
held = held || (f.Kind == kind && f.ID == kind)
|
||||
}
|
||||
if !held {
|
||||
return
|
||||
}
|
||||
if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil {
|
||||
logf("the finding %s is not cleared: %v", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,14 +92,9 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste
|
||||
return s, mem, q, said, waits, asks, view, Verbs{Hearer: h, Starter: s, Acts: &actLog{}}
|
||||
}
|
||||
|
||||
// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a
|
||||
// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started
|
||||
// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places
|
||||
// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline
|
||||
// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed.
|
||||
func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
s, mem, q, said, waits, asks, view, v := queueFixture()
|
||||
take := func(source int64, modules ...string) {
|
||||
// takeIn hears a take-in.taken of main at source, its entries the modules named.
|
||||
func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) {
|
||||
return func(source int64, modules ...string) {
|
||||
t.Helper()
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
|
||||
for _, m := range modules {
|
||||
@@ -109,6 +104,16 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
t.Fatalf("the take-in of source %d started %d (%v)", source, n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a
|
||||
// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started
|
||||
// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places
|
||||
// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline
|
||||
// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed.
|
||||
func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
s, mem, q, said, waits, asks, view, v := queueFixture()
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
first := len(asks.ids)
|
||||
if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 {
|
||||
@@ -176,16 +181,7 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
// pipeline moving the same module runs to its end.
|
||||
func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) {
|
||||
s, mem, q, said, waits, _, view, _ := queueFixture()
|
||||
take := func(source int64, modules ...string) {
|
||||
t.Helper()
|
||||
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
|
||||
for _, m := range modules {
|
||||
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
|
||||
}
|
||||
if n, err := s.TakenIn(ev, view); n != 1 || err != nil {
|
||||
t.Fatalf("the take-in of source %d started %d (%v)", source, n, err)
|
||||
}
|
||||
}
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "app", "lib")
|
||||
take(6, "app")
|
||||
@@ -213,3 +209,64 @@ func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R12/AC1 to AC3 (design 47 H §748; H-A-14, H-A-52, H-A-106): a start beyond pipelines-waiting-cap still waits and raises
|
||||
// queue-full naming it, cleared when an end brings the waiting back to the cap; a pipeline whose every step not ended is an
|
||||
// approval asked of the operator leaves the bound, the next placed, and once answered and still running it is keyed running
|
||||
// at once, over the bound; pipelines waiting with none running raise pipelines-stalled naming the first, cleared when one
|
||||
// runs. The findings are the scope delivery's, their kind their id, warnings with no owner.
|
||||
func TestTheQueuesApprovalStallAndCap(t *testing.T) {
|
||||
s, mem, q, _, _, _, view, v := queueFixture()
|
||||
f := newHeld()
|
||||
s.Queue.Findings, s.Queue.Cap = f, 1
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "lib")
|
||||
if len(f.held) != 0 {
|
||||
t.Fatalf("findings %v with one waiting at a cap of 1", f.held)
|
||||
}
|
||||
take(6, "tool")
|
||||
full, ok := f.held["delivery.queue-full.queue-full"]
|
||||
if p, _, _ := mem.Get("main.6"); !ok || p.Outcome != seat.PipelineOutcomeWaiting || !strings.Contains(full.Summary, "main.6") ||
|
||||
full.Severity != "warning" || full.Owner != "" {
|
||||
t.Fatalf("over the cap main.6 = %s, queue-full %+v (%v)", p.Outcome, full, ok)
|
||||
}
|
||||
// main.4 waits on its approval alone.
|
||||
p4, rev, _ := mem.Get("main.4")
|
||||
for i := range p4.Steps {
|
||||
st := &p4.Steps[i]
|
||||
st.Outcome = seat.StepResultOutcomePassed
|
||||
if st.Step == seat.StepResultStepApproval {
|
||||
st.Outcome, st.Ask = seat.StepResultOutcomeRunning, "the-ask"
|
||||
}
|
||||
}
|
||||
mem.Update("main.4", p4, rev)
|
||||
if err := v.Hearer.said(p4, nil, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, keyed := q["running.main.4"]
|
||||
if p, _, _ := mem.Get("main.5"); keyed || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Errorf("with main.4 on its approval: main.4 keyed %v, main.5 %s; want main.4 out of the bound and main.5 placed", keyed, p.Outcome)
|
||||
}
|
||||
if _, ok := f.held["delivery.queue-full.queue-full"]; ok {
|
||||
t.Error("queue-full stands with one waiting at a cap of 1")
|
||||
}
|
||||
s.Rejoin("main.4")
|
||||
if running, _ := s.Queue.Running(); len(running) != 2 {
|
||||
t.Errorf("after main.4's answer, running %v; want main.4 keyed at once beside main.5", running)
|
||||
}
|
||||
// None runs while main.6 waits: pipelines-stalled names it, cleared once one runs.
|
||||
for _, k := range []string{"running.main.4", "running.main.5"} {
|
||||
q.Delete(k)
|
||||
}
|
||||
s.Queue.judge(t.Logf)
|
||||
if st, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; !ok || !strings.Contains(st.Summary, "main.6") || st.Owner != "" {
|
||||
t.Errorf("pipelines-stalled = %+v (%v), want it naming main.6", st, ok)
|
||||
}
|
||||
if n, err := s.PlaceQueued(view); n != 1 || err != nil {
|
||||
t.Fatalf("placed %d (%v)", n, err)
|
||||
}
|
||||
if _, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; ok {
|
||||
t.Error("pipelines-stalled stands with main.6 running")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,7 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", branch, key, err)
|
||||
}
|
||||
s.Queue.overCap(key, s.logf)
|
||||
}
|
||||
first := in.Members[0]
|
||||
last := first.Commit
|
||||
|
||||
@@ -457,6 +457,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
|
||||
}
|
||||
s.Queue.overCap(key, s.logf)
|
||||
return true, nil
|
||||
}
|
||||
s.ran(key)
|
||||
|
||||
@@ -158,6 +158,9 @@ type Hearer struct {
|
||||
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
|
||||
// Starter.Turn); nil takes none.
|
||||
Turn func(key string)
|
||||
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
|
||||
// nil keys none.
|
||||
Rejoin func(key string)
|
||||
}
|
||||
|
||||
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
|
||||
@@ -414,6 +417,11 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
||||
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
|
||||
}
|
||||
}
|
||||
// A pipeline waiting on the operator's approval alone does not count against pipelines-at-once (§748, H-A-106 (b)): its
|
||||
// running key goes and the next waiting is placed, as at an end.
|
||||
if !ended && h.Turn != nil && awaitsApprovalOnly(p) {
|
||||
h.Turn(p.Pipeline)
|
||||
}
|
||||
if ended {
|
||||
h.Failures.Ended(p)
|
||||
err := h.say(seat.EventPipelineEnded, p, nil)
|
||||
|
||||
Reference in New Issue
Block a user