R12: the queue's approval wait, stall and cap

A pipeline whose every step not ended is an approval asked of the operator leaves the bound: its
running key goes and the next waiting is placed; answered and still running, it is keyed running
at once, over the bound. After every placing, pipelines waiting with none running raise the finding
pipelines-stalled naming the first waiting, cleared once one runs. A start beyond
pipelines-waiting-cap is an ordinary waiting pipeline and raises queue-full naming it, cleared when
an end brings the waiting back to the cap. Both findings are the scope delivery's, the kind as id,
warnings with no owner.

Builds: design 47 H §748; lead answers H-A-14, H-A-36, H-A-52, H-A-106

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent bbdd1741fa
commit bb930de377
7 changed files with 226 additions and 19 deletions
@@ -207,7 +207,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
logger.Printf("the queue: %d pipelines at once, at most %d waiting", atOnce, waitingCap)
starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce}
starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce, Cap: waitingCap,
Findings: rv.Findings, Observer: cred.Module}
hearer.Rejoin = starter.Rejoin
hearer.Turn = func(key string) {
if n, err := starter.Turn(key, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the end of %s placed %d waiting pipelines (%v)", key, n, err)
@@ -235,6 +235,11 @@ func (h *Hearer) OnDecided(event oc.Warrant) (seat.Pipeline, error) {
}
}
h.appendUnlogged(key)
// Answered and still running on more than approvals asked, it is keyed running at once, over the bound (§748,
// H-A-106 (b)).
if !ended && h.Rejoin != nil && p.Outcome == seat.PipelineOutcomeRunning && !awaitsApprovalOnly(p) {
h.Rejoin(key)
}
return p, h.said(p, nil, ended)
}
}
@@ -1,6 +1,7 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
@@ -8,6 +9,8 @@ import (
"strconv"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
@@ -33,6 +36,13 @@ type QueueStore interface {
type Queue struct {
Store QueueStore
AtOnce int
// Cap is pipelines-waiting-cap: a start beyond it still waits, and raises queue-full (H-A-106 (a)); 0 holds no cap.
Cap int
// Findings writes the queue's findings pipelines-stalled and queue-full, Observer the runner's own module; nil writes
// none.
Findings Findings
Observer string
Now func() time.Time
}
const (
@@ -243,6 +253,7 @@ func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error
if s.Queue == nil {
return 0, nil
}
defer s.Queue.judge(s.logf)
placedN := 0
for {
full, err := s.Queue.full()
@@ -310,3 +321,125 @@ func (s *Starter) dropWaiting(wkey string) error {
}
return nil
}
// Rejoin writes the running key of pipeline key whose approval was answered and which still runs, at once and over the
// bound, since it asks nothing new but its end (design 47 H §748, H-A-106 (b)).
func (s *Starter) Rejoin(key string) {
if s.Queue == nil {
return
}
if ok, err := s.Queue.run(key); err != nil {
s.logf("%s runs again after its approval, and %v", key, err)
} else if ok {
s.logf("%s runs again after its approval: keyed running at once", key)
}
s.Queue.judge(s.logf)
}
// awaitsApprovalOnly says whether every step of p not ended is an approval asked of the operator (H-A-106 (b)): such a
// pipeline does not count against pipelines-at-once (§748).
func awaitsApprovalOnly(p seat.Pipeline) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
asked := false
for _, st := range p.Steps {
if !open(st) && st.Outcome != seat.StepResultOutcomeWaiting {
continue
}
if st.Step != seat.StepResultStepApproval || st.Outcome != seat.StepResultOutcomeRunning || st.Ask == "" {
return false
}
asked = true
}
return asked
}
// The queue's findings (design 47 H §748, H-A-52, H-A-106 (c)): scope delivery, one each for the mesh, the kind as id.
const (
PipelinesStalledKind = "pipelines-stalled"
QueueFullKind = "queue-full"
)
// waitingCount is how many pipelines wait, -1 when the queue is not read.
func (q *Queue) waitingCount() int {
ks, err := q.Store.Keys(waitingPrefix)
if err != nil {
return -1
}
return len(ks)
}
// overCap raises queue-full naming pipeline when more pipelines wait than pipelines-waiting-cap (H-A-106 (a)).
func (q *Queue) overCap(pipeline string, logf func(string, ...any)) {
if q.Cap <= 0 {
return
}
if n := q.waitingCount(); n > q.Cap {
q.raise(QueueFullKind, fmt.Sprintf("%d pipelines wait, over pipelines-waiting-cap %d; the newest is %s", n, q.Cap, pipeline),
"More pipelines wait than the queue's cap",
fmt.Sprintf("The start of %s was written waiting beyond pipelines-waiting-cap %d: %d pipelines wait their turn under pipelines-at-once %d. It is cleared when an end brings the waiting back to the cap.", pipeline, q.Cap, n, q.AtOnce),
"fewer pipelines waiting than pipelines-waiting-cap", logf)
}
}
// judge raises pipelines-stalled naming the first waiting when pipelines wait and none runs, clearing it when one runs or
// none waits; and clears queue-full when the waiting are back to the cap or below (§748, H-A-106).
func (q *Queue) judge(logf func(string, ...any)) {
if q.Findings == nil {
return
}
running, err := q.Running()
if err != nil {
logf("the queue is not judged: %v", err)
return
}
_, waiting, err := q.Waiting()
if err != nil {
logf("the queue is not judged: %v", err)
return
}
if len(waiting) > 0 && len(running) == 0 {
q.raise(PipelinesStalledKind, fmt.Sprintf("%d pipelines wait and none runs; the first is %s", len(waiting), waiting[0]),
"Pipelines wait and none runs",
fmt.Sprintf("After a placing, %d pipelines wait their turn under pipelines-at-once %d and none runs; the first waiting is %s. It is cleared when one runs.", len(waiting), q.AtOnce, waiting[0]),
"a pipeline running", logf)
} else {
q.clear(PipelinesStalledKind, "a pipeline runs, or none waits", logf)
}
if q.Cap <= 0 || len(waiting) <= q.Cap {
q.clear(QueueFullKind, fmt.Sprintf("%d pipelines wait, within pipelines-waiting-cap %d", len(waiting), q.Cap), logf)
}
}
func (q *Queue) raise(kind, summary, headline, explanation, needs string, logf func(string, ...any)) {
if q.Findings == nil {
return
}
now := time.Now().UTC()
if q.Now != nil {
now = q.Now().UTC()
}
err := q.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeDelivery, ID: kind, Kind: kind,
Severity: condseat.FindingSeverityWarning, Summary: summary, Headline: headline, Explanation: explanation, Needs: needs,
Source: "pipeline-runner." + kind, Observer: q.Observer, At: now, Subject: condseat.Subject{Scope: mcc.ScopeDelivery, ID: kind}})
if err != nil {
logf("the finding %s is not written: %v", kind, err)
}
}
func (q *Queue) clear(kind, why string, logf func(string, ...any)) {
if q.Findings == nil {
return
}
held := false
for _, f := range q.Findings.Held() {
held = held || (f.Kind == kind && f.ID == kind)
}
if !held {
return
}
if err := q.Findings.Clear(context.Background(), mcc.ScopeDelivery, kind, kind, why); err != nil {
logf("the finding %s is not cleared: %v", kind, err)
}
}
@@ -92,14 +92,9 @@ func queueFixture() (*Starter, memStore, memQueue, *[]string, *[]seat.Wait, *ste
return s, mem, q, said, waits, asks, view, Verbs{Hearer: h, Starter: s, Acts: &actLog{}}
}
// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a
// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started
// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places
// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline
// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed.
func TestThePipelinesWaitTheirTurn(t *testing.T) {
s, mem, q, said, waits, asks, view, v := queueFixture()
take := func(source int64, modules ...string) {
// takeIn hears a take-in.taken of main at source, its entries the modules named.
func takeIn(t *testing.T, s *Starter, view func(string) (Half, bool)) func(source int64, modules ...string) {
return func(source int64, modules ...string) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
for _, m := range modules {
@@ -109,6 +104,16 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
t.Fatalf("the take-in of source %d started %d (%v)", source, n, err)
}
}
}
// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a
// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started
// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places
// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline
// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed.
func TestThePipelinesWaitTheirTurn(t *testing.T) {
s, mem, q, said, waits, asks, view, v := queueFixture()
take := takeIn(t, s, view)
take(4, "app")
first := len(asks.ids)
if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 {
@@ -176,16 +181,7 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
// pipeline moving the same module runs to its end.
func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) {
s, mem, q, said, waits, _, view, _ := queueFixture()
take := func(source int64, modules ...string) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7}
for _, m := range modules {
ev.Entries = append(ev.Entries, catalog.TakeInEntry{Module: m, Commit: "c1", Fingerprint: "src2:" + m})
}
if n, err := s.TakenIn(ev, view); n != 1 || err != nil {
t.Fatalf("the take-in of source %d started %d (%v)", source, n, err)
}
}
take := takeIn(t, s, view)
take(4, "app")
take(5, "app", "lib")
take(6, "app")
@@ -213,3 +209,64 @@ func TestANewerWaitingPipelineSupersedesTheOlder(t *testing.T) {
}
}
}
// R12/AC1 to AC3 (design 47 H §748; H-A-14, H-A-52, H-A-106): a start beyond pipelines-waiting-cap still waits and raises
// queue-full naming it, cleared when an end brings the waiting back to the cap; a pipeline whose every step not ended is an
// approval asked of the operator leaves the bound, the next placed, and once answered and still running it is keyed running
// at once, over the bound; pipelines waiting with none running raise pipelines-stalled naming the first, cleared when one
// runs. The findings are the scope delivery's, their kind their id, warnings with no owner.
func TestTheQueuesApprovalStallAndCap(t *testing.T) {
s, mem, q, _, _, _, view, v := queueFixture()
f := newHeld()
s.Queue.Findings, s.Queue.Cap = f, 1
take := takeIn(t, s, view)
take(4, "app")
take(5, "lib")
if len(f.held) != 0 {
t.Fatalf("findings %v with one waiting at a cap of 1", f.held)
}
take(6, "tool")
full, ok := f.held["delivery.queue-full.queue-full"]
if p, _, _ := mem.Get("main.6"); !ok || p.Outcome != seat.PipelineOutcomeWaiting || !strings.Contains(full.Summary, "main.6") ||
full.Severity != "warning" || full.Owner != "" {
t.Fatalf("over the cap main.6 = %s, queue-full %+v (%v)", p.Outcome, full, ok)
}
// main.4 waits on its approval alone.
p4, rev, _ := mem.Get("main.4")
for i := range p4.Steps {
st := &p4.Steps[i]
st.Outcome = seat.StepResultOutcomePassed
if st.Step == seat.StepResultStepApproval {
st.Outcome, st.Ask = seat.StepResultOutcomeRunning, "the-ask"
}
}
mem.Update("main.4", p4, rev)
if err := v.Hearer.said(p4, nil, false); err != nil {
t.Fatal(err)
}
_, keyed := q["running.main.4"]
if p, _, _ := mem.Get("main.5"); keyed || p.Outcome != seat.PipelineOutcomeRunning {
t.Errorf("with main.4 on its approval: main.4 keyed %v, main.5 %s; want main.4 out of the bound and main.5 placed", keyed, p.Outcome)
}
if _, ok := f.held["delivery.queue-full.queue-full"]; ok {
t.Error("queue-full stands with one waiting at a cap of 1")
}
s.Rejoin("main.4")
if running, _ := s.Queue.Running(); len(running) != 2 {
t.Errorf("after main.4's answer, running %v; want main.4 keyed at once beside main.5", running)
}
// None runs while main.6 waits: pipelines-stalled names it, cleared once one runs.
for _, k := range []string{"running.main.4", "running.main.5"} {
q.Delete(k)
}
s.Queue.judge(t.Logf)
if st, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; !ok || !strings.Contains(st.Summary, "main.6") || st.Owner != "" {
t.Errorf("pipelines-stalled = %+v (%v), want it naming main.6", st, ok)
}
if n, err := s.PlaceQueued(view); n != 1 || err != nil {
t.Fatalf("placed %d (%v)", n, err)
}
if _, ok := f.held["delivery.pipelines-stalled.pipelines-stalled"]; ok {
t.Error("pipelines-stalled stands with main.6 running")
}
}
@@ -99,6 +99,7 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err
if _, err := s.supersedeWaiting(p); err != nil {
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", branch, key, err)
}
s.Queue.overCap(key, s.logf)
}
first := in.Members[0]
last := first.Commit
@@ -457,6 +457,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
if _, err := s.supersedeWaiting(p); err != nil {
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
}
s.Queue.overCap(key, s.logf)
return true, nil
}
s.ran(key)
@@ -158,6 +158,9 @@ type Hearer struct {
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
// Starter.Turn); nil takes none.
Turn func(key string)
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
// nil keys none.
Rejoin func(key string)
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
@@ -414,6 +417,11 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
}
}
// A pipeline waiting on the operator's approval alone does not count against pipelines-at-once (§748, H-A-106 (b)): its
// running key goes and the next waiting is placed, as at an end.
if !ended && h.Turn != nil && awaitsApprovalOnly(p) {
h.Turn(p.Pipeline)
}
if ended {
h.Failures.Ended(p)
err := h.say(seat.EventPipelineEnded, p, nil)