From bb930de3771f21ab87a2fbbe23ed8b923eef034c Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 04:10:54 +0100 Subject: [PATCH] R12: the queue's approval wait, stall and cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A pipeline whose every step not ended is an approval asked of the operator leaves the bound: its running key goes and the next waiting is placed; answered and still running, it is keyed running at once, over the bound. After every placing, pipelines waiting with none running raise the finding pipelines-stalled naming the first waiting, cleared once one runs. A start beyond pipelines-waiting-cap is an ordinary waiting pipeline and raises queue-full naming it, cleared when an end brings the waiting back to the cap. Both findings are the scope delivery's, the kind as id, warnings with no owner. Builds: design 47 H §748; lead answers H-A-14, H-A-36, H-A-52, H-A-106 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-pipeline/main.go | 4 +- .../internal/pipeline/approval.go | 5 + .../internal/pipeline/queue.go | 133 ++++++++++++++++++ .../internal/pipeline/queue_test.go | 93 +++++++++--- .../internal/pipeline/run.go | 1 + .../internal/pipeline/start.go | 1 + .../internal/pipeline/steps.go | 8 ++ 7 files changed, 226 insertions(+), 19 deletions(-) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 479018dd..10d9e64c 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -207,7 +207,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } logger.Printf("the queue: %d pipelines at once, at most %d waiting", atOnce, waitingCap) - starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce} + starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce, Cap: waitingCap, + Findings: rv.Findings, Observer: cred.Module} + hearer.Rejoin = starter.Rejoin hearer.Turn = func(key string) { if n, err := starter.Turn(key, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the end of %s placed %d waiting pipelines (%v)", key, n, err) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go index b4e8bdf2..6e706b17 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/approval.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/approval.go @@ -235,6 +235,11 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) { } } h.appendUnlogged(key) + // Answered and still running on more than approvals asked, it is keyed running at once, over the bound (§748, + // H-A-106 (b)). + if !ended && h.Rejoin != nil && p.Outcome == seat.PipelineOutcomeRunning && !awaitsApprovalOnly(p) { + h.Rejoin(key) + } return p, h.said(p, nil, ended) } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index 86755f00..855191db 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -1,6 +1,7 @@ package pipeline import ( + "context" "encoding/json" "errors" "fmt" @@ -8,6 +9,8 @@ import ( "strconv" "time" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) @@ -33,6 +36,13 @@ type QueueStore interface { type Queue struct { Store QueueStore AtOnce int + // Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap. + Cap int + // Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes + // none. + Findings Findings + Observer string + Now func() time.Time } const ( @@ -243,6 +253,7 @@ func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error if s.Queue == nil { return 0, nil } + defer s.Queue.judge(s.logf) placedN := 0 for { full, err := s.Queue.full() @@ -310,3 +321,125 @@ func (s *Starter) dropWaiting(wkey string) error { } return nil } + +// Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the +// bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)). +func (s *Starter) Rejoin(key string) { + if s.Queue == nil { + return + } + if ok, err := s.Queue.run(key); err != nil { + s.logf("%s runs again after its approval, and %v", key, err) + } else if ok { + s.logf("%s runs again after its approval: keyed running at once", key) + } + s.Queue.judge(s.logf) +} + +// awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a +// pipeline does not count against pipelines-at-once (§748). +func awaitsApprovalOnly(p seat.Pipeline) bool { + if p.Outcome != seat.PipelineOutcomeRunning { + return false + } + asked := false + for _, st := range p.Steps { + if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting { + continue + } + if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" { + return false + } + asked = true + } + return asked +} + +// The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id. +const ( + PipelinesStalledKind = "pipelines-stalled" + QueueFullKind = "queue-full" +) + +// waitingCount is how many pipelines wait, -1 when the queue is not read. +func (q *Queue) waitingCount() int { + ks, err := q.Store.Keys(waitingPrefix) + if err != nil { + return -1 + } + return len(ks) +} + +// overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)). +func (q *Queue) overCap(pipeline string, logf func(string, ...any)) { + if q.Cap <= 0 { + return + } + if n := q.waitingCount(); n > q.Cap { + q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline), + "More pipelines wait than the queue's cap", + fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce), + "fewer pipelines waiting than pipelines-waiting-cap", logf) + } +} + +// judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or +// none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106). +func (q *Queue) judge(logf func(string, ...any)) { + if q.Findings == nil { + return + } + running, err := q.Running() + if err != nil { + logf("the queue is not judged: %v", err) + return + } + _, waiting, err := q.Waiting() + if err != nil { + logf("the queue is not judged: %v", err) + return + } + if len(waiting) > 0 && len(running) == 0 { + q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]), + "Pipelines wait and none runs", + fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]), + "a pipeline running", logf) + } else { + q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf) + } + if q.Cap <= 0 || len(waiting) <= q.Cap { + q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf) + } +} + +func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) { + if q.Findings == nil { + return + } + now := time.Now().UTC() + if q.Now != nil { + now = q.Now().UTC() + } + err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind, + Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs, + Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}}) + if err != nil { + logf("the finding %s is not written: %v", kind, err) + } +} + +func (q *Queue) clear(kind, why string, logf func(string, ...any)) { + if q.Findings == nil { + return + } + held := false + for _, f := range q.Findings.Held() { + held = held || (f.Kind == kind && f.ID == kind) + } + if !held { + return + } + if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil { + logf("the finding %s is not cleared: %v", kind, err) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go index d964de37..cbc19e30 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -92,14 +92,9 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste return s, mem, q, said, waits, asks, view, Verbs{Hearer: h, Starter: s, Acts: &actLog{}} } -// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a -// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started -// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places -// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline -// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed. -func TestThePipelinesWaitTheirTurn(t *testing.T) { - s, mem, q, said, waits, asks, view, v := queueFixture() - take := func(source int64, modules ...string) { +// takeIn hears a take-in.taken of main at source, its entries the modules named. +func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) { + return func(source int64, modules ...string) { t.Helper() ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} for _, m := range modules { @@ -109,6 +104,16 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { t.Fatalf("the take-in of source %d started %d (%v)", source, n, err) } } +} + +// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a +// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started +// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places +// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline +// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed. +func TestThePipelinesWaitTheirTurn(t *testing.T) { + s, mem, q, said, waits, asks, view, v := queueFixture() + take := takeIn(t, s, view) take(4, "app") first := len(asks.ids) if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 { @@ -176,16 +181,7 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) { // pipeline moving the same module runs to its end. func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) { s, mem, q, said, waits, _, view, _ := queueFixture() - take := func(source int64, modules ...string) { - t.Helper() - ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7} - for _, m := range modules { - ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m}) - } - if n, err := s.TakenIn(ev, view); n != 1 || err != nil { - t.Fatalf("the take-in of source %d started %d (%v)", source, n, err) - } - } + take := takeIn(t, s, view) take(4, "app") take(5, "app", "lib") take(6, "app") @@ -213,3 +209,64 @@ func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) { } } } + +// R12/AC1 to AC3 (design 47 H §748; H-A-14, H-A-52, H-A-106): a start beyond pipelines-waiting-cap still waits and raises +// queue-full naming it, cleared when an end brings the waiting back to the cap; a pipeline whose every step not ended is an +// approval asked of the operator leaves the bound, the next placed, and once answered and still running it is keyed running +// at once, over the bound; pipelines waiting with none running raise pipelines-stalled naming the first, cleared when one +// runs. The findings are the scope delivery's, their kind their id, warnings with no owner. +func TestTheQueuesApprovalStallAndCap(t *testing.T) { + s, mem, q, _, _, _, view, v := queueFixture() + f := newHeld() + s.Queue.Findings, s.Queue.Cap = f, 1 + take := takeIn(t, s, view) + take(4, "app") + take(5, "lib") + if len(f.held) != 0 { + t.Fatalf("findings %v with one waiting at a cap of 1", f.held) + } + take(6, "tool") + full, ok := f.held["delivery.queue-full.queue-full"] + if p, _, _ := mem.Get("main.6"); !ok || p.Outcome != seat.PipelineOutcomeWaiting || !strings.Contains(full.Summary, "main.6") || + full.Severity != "warning" || full.Owner != "" { + t.Fatalf("over the cap main.6 = %s, queue-full %+v (%v)", p.Outcome, full, ok) + } + // main.4 waits on its approval alone. + p4, rev, _ := mem.Get("main.4") + for i := range p4.Steps { + st := &p4.Steps[i] + st.Outcome = seat.StepResultOutcomePassed + if st.Step == seat.StepResultStepApproval { + st.Outcome, st.Ask = seat.StepResultOutcomeRunning, "the-ask" + } + } + mem.Update("main.4", p4, rev) + if err := v.Hearer.said(p4, nil, false); err != nil { + t.Fatal(err) + } + _, keyed := q["running.main.4"] + if p, _, _ := mem.Get("main.5"); keyed || p.Outcome != seat.PipelineOutcomeRunning { + t.Errorf("with main.4 on its approval: main.4 keyed %v, main.5 %s; want main.4 out of the bound and main.5 placed", keyed, p.Outcome) + } + if _, ok := f.held["delivery.queue-full.queue-full"]; ok { + t.Error("queue-full stands with one waiting at a cap of 1") + } + s.Rejoin("main.4") + if running, _ := s.Queue.Running(); len(running) != 2 { + t.Errorf("after main.4's answer, running %v; want main.4 keyed at once beside main.5", running) + } + // None runs while main.6 waits: pipelines-stalled names it, cleared once one runs. + for _, k := range []string{"running.main.4", "running.main.5"} { + q.Delete(k) + } + s.Queue.judge(t.Logf) + if st, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; !ok || !strings.Contains(st.Summary, "main.6") || st.Owner != "" { + t.Errorf("pipelines-stalled = %+v (%v), want it naming main.6", st, ok) + } + if n, err := s.PlaceQueued(view); n != 1 || err != nil { + t.Fatalf("placed %d (%v)", n, err) + } + if _, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; ok { + t.Error("pipelines-stalled stands with main.6 running") + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index f250c105..04c6b456 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -99,6 +99,7 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err if _, err := s.supersedeWaiting(p); err != nil { s.logf("the older waiting pipelines of %s are not superseded by %s: %v", branch, key, err) } + s.Queue.overCap(key, s.logf) } first := in.Members[0] last := first.Commit diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 9d3114c8..05f9a1bd 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -457,6 +457,7 @@ func (s *Starter) Start(b Begin) (bool, error) { if _, err := s.supersedeWaiting(p); err != nil { s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err) } + s.Queue.overCap(key, s.logf) return true, nil } s.ran(key) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 96c4cd8e..2d3809fb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -158,6 +158,9 @@ type Hearer struct { // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; // Starter.Turn); nil takes none. Turn func(key string) + // Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin); + // nil keys none. + Rejoin func(key string) } // EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps @@ -414,6 +417,11 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err) } } + // A pipeline waiting on the operator's approval alone does not count against pipelines-at-once (§748, H-A-106 (b)): its + // running key goes and the next waiting is placed, as at an end. + if !ended && h.Turn != nil && awaitsApprovalOnly(p) { + h.Turn(p.Pipeline) + } if ended { h.Failures.Ended(p) err := h.say(seat.EventPipelineEnded, p, nil)