R11: at most pipelines-at-once pipelines run; the others wait in order

The runner's settings pipelines-at-once (default 1, from 1 to 8) and pipelines-waiting-cap (default
64) are written to files by the manifest and read at the start; its state queue holds queue.next,
waiting.<sequence> and running.<pipeline>. A start (a take-in's, a publish's, a hand start's, a run
on request's) judged against the running keys alone either runs, its running key written after its
pipeline, or is written waiting under the next sequence, its waiting key first, with no step asked,
saying pipeline-started and wait-held step pipeline on pipelines-at-once. Every end, a cancel's too,
deletes the running key and places the lowest sequence, which says wait-ended and has its steps
placed (a run's from its candidate's newest revision). At the start a running key whose pipeline
runs no more is deleted and the queue placed. The test runs' whole test sets pipelines-at-once 8, as
its candidates run side by side.

Builds: design 47 H §748; lead answers H-A-14, H-A-36, H-A-52, H-A-53, H-A-60, H-A-72, H-A-96

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:31 +01:00
parent cf703c2a7b
commit 63bb53bb6b
12 changed files with 728 additions and 14 deletions
@@ -196,6 +196,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
registrar.Store(&register)
starter.Store = pipelineStore{ctx: ctx, kv: pipelines}
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
// the state queue, placed at the start from it and on every end.
atOnce, waitingCap, err := queueSettings(getenv)
if err != nil {
return err
}
queueBucket, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StateQueue), "keep the pipelines waiting their turn")
if err != nil {
return err
}
logger.Printf("the queue: %d pipelines at once, at most %d waiting", atOnce, waitingCap)
starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce}
hearer.Turn = func(key string) {
if n, err := starter.Turn(key, rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the end of %s placed %d waiting pipelines (%v)", key, n, err)
}
}
starter.RunTurn = func(u pipeline.UnplacedRun) {
one := func() ([]pipeline.UnplacedRun, error) { return []pipeline.UnplacedRun{u}, nil }
if err := placeLostRuns(one, calls.CallKept, placeHalf, logger.Printf); err != nil {
logger.Printf("the run %s whose turn came is not placed: %v", u.Run, err)
}
}
// A run's placing lost while the runner was down is taken up from its candidate's newest revision, once the registrations
// that place its steps are held (A-222).
go func() {
@@ -212,6 +235,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := placeLostRuns(starter.UnplacedRuns, calls.CallKept, placeHalf, logger.Printf); err != nil {
logger.Printf("the runs not placed at the start: %v", err)
}
// The queue is placed at the start from its own state (§748).
if n, err := starter.StartQueue(rv.Catalog.View); err != nil || n > 0 {
logger.Printf("the queue at the start: %d waiting pipelines placed (%v)", n, err)
}
}()
// A run's candidate revision places its pipelines' steps, its entries read through the seat's revision verb (A-180).
place := func(ev catalog.CatalogRevisionEvent) error {
@@ -0,0 +1,102 @@
package main
import (
"context"
"errors"
"fmt"
"slices"
"strconv"
"strings"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-pipeline/internal/pipeline"
)
const (
// envPipelinesAtOnce and envPipelinesWaitingCap name the files of the settings pipelines-at-once and
// pipelines-waiting-cap (design 47 H §748).
envPipelinesAtOnce = "MESH_RUNNER_PIPELINES_AT_ONCE_FILE"
envPipelinesWaitingCap = "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE"
defaultPipelinesAtOnce = 1
maxPipelinesAtOnce = 8
defaultWaitingCap = 64
)
// queueSettings reads pipelines-at-once (default 1, from 1 to 8) and pipelines-waiting-cap (default 64) from their files
// (design 47 H §748): a file missing or empty is the default, any other value outside its range an error naming the file.
func queueSettings(getenv func(string) string) (int, int, error) {
read := func(env string, def, lo, hi int) (int, error) {
v, err := settingText(getenv(env))
if err != nil || v == "" {
return def, err
}
n, err := strconv.Atoi(v)
if err != nil || n < lo || (hi > 0 && n > hi) {
return 0, fmt.Errorf("%s holds %q, which is not a whole number in range (at least %d, at most %d where above 0)", getenv(env), v, lo, hi)
}
return n, nil
}
atOnce, err := read(envPipelinesAtOnce, defaultPipelinesAtOnce, 1, maxPipelinesAtOnce)
if err != nil {
return 0, 0, err
}
waitingCap, err := read(envPipelinesWaitingCap, defaultWaitingCap, 1, 0)
return atOnce, waitingCap, err
}
// queueKV is the runner's state queue on the bus, written by compare-and-set (design 47 H §748).
type queueKV struct {
ctx context.Context
kv jetstream.KeyValue
}
func (q queueKV) Get(key string) ([]byte, uint64, error) {
e, err := q.kv.Get(q.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, pipeline.ErrNoKey
} else if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (q queueKV) Create(key string, value []byte) (uint64, error) {
rev, err := q.kv.Create(q.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, pipeline.ErrExists
}
return rev, err
}
func (q queueKV) Update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := q.kv.Update(q.ctx, key, value, revision)
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
return 0, pipeline.ErrChanged
}
return rev, err
}
func (q queueKV) Delete(key string) error {
err := q.kv.Delete(q.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return pipeline.ErrNoKey
}
return err
}
func (q queueKV) Keys(prefix string) ([]string, error) {
lister, err := q.kv.ListKeysFiltered(q.ctx, prefix+">")
if err != nil {
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return nil, err
}
var out []string
for k := range lister.Keys() {
out = append(out, k)
}
slices.Sort(out)
return out, nil
}
@@ -60,7 +60,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
@@ -58,7 +58,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
defer cancel()
const module = "mesh-delivery-pipeline"
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} {
for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil {
t.Fatal(err)
}
@@ -186,9 +186,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
return out
}
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds.
// The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds; the
// candidates' pipelines run side by side, so pipelines-at-once is its most, 8 (§748).
dir := t.TempDir()
files := map[string]string{envRunsCap: "1", envStepBound: "2s"}
files := map[string]string{envRunsCap: "1", envStepBound: "2s", envPipelinesAtOnce: "8"}
env := map[string]string{}
for name, content := range files {
path := filepath.Join(dir, name)
@@ -131,4 +131,28 @@ func TestWidensSettings(t *testing.T) {
}
})
}
// R11 (design 47 H §748): pipelines-at-once, default 1, from 1 to 8; pipelines-waiting-cap, default 64.
for _, c := range []struct {
name string
files map[string]string
atOnce, limit int
ok bool
}{
{"queue defaults", nil, 1, 64, true},
{"queue set", map[string]string{envPipelinesAtOnce: "3\n", envPipelinesWaitingCap: "10"}, 3, 10, true},
{"over 8 at once", map[string]string{envPipelinesAtOnce: "9"}, 0, 0, false},
{"none at once", map[string]string{envPipelinesAtOnce: "0"}, 0, 0, false},
{"no cap", map[string]string{envPipelinesWaitingCap: "0"}, 1, 0, false},
} {
t.Run(c.name, func(t *testing.T) {
env := map[string]string{}
for k, v := range c.files {
env[k] = write(c.name+k, v)
}
atOnce, limit, err := queueSettings(func(k string) string { return env[k] })
if (err == nil) != c.ok || (c.ok && (atOnce != c.atOnce || limit != c.limit)) {
t.Fatalf("%d %d, %v", atOnce, limit, err)
}
})
}
}
@@ -0,0 +1,312 @@
package pipeline
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strconv"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// ErrNoKey is a QueueStore's answer to a Get of a key it does not hold.
var ErrNoKey = errors.New("the queue holds no such key")
// QueueStore is the runner's state queue (design 47 H §748, H-A-36, H-A-72 (2), H-A-96): its counter queue.next, one key
// waiting.<sequence> per waiting pipeline and one key running.<pipeline> per pipeline whose steps are asked. A write is by
// compare-and-set: Create on a key's absence (ErrExists when it is written), Update on the revision read (ErrChanged).
type QueueStore interface {
Get(key string) ([]byte, uint64, error)
Create(key string, value []byte) (uint64, error)
Update(key string, value []byte, revision uint64) (uint64, error)
Delete(key string) error
// Keys are the keys beginning with prefix, in key order.
Keys(prefix string) ([]string, error)
}
// Queue bounds the pipelines whose steps are asked at once to AtOnce, the setting pipelines-at-once (design 47 H §748,
// H-A-14): it is judged against the running.<pipeline> keys alone (H-A-96); a pipeline over it waits under the next value
// of queue.next.
type Queue struct {
Store QueueStore
AtOnce int
}
const (
runningPrefix = "running."
waitingPrefix = "waiting."
)
// sequenceKey is a waiting pipeline's key, its sequence 20 digits, so the lowest is read first by key order (H-A-72 (2)).
func sequenceKey(seq int64) string {
return seat.QueueByWaitingSequenceKey(fmt.Sprintf("%020d", seq))
}
// Running are the pipelines whose running.<pipeline> key is written.
func (q *Queue) Running() ([]string, error) {
ks, err := q.Store.Keys(runningPrefix)
if err != nil {
return nil, fmt.Errorf("the queue's running pipelines are not read: %w", err)
}
out := make([]string, 0, len(ks))
for _, k := range ks {
if p, ok := seat.ParseQueueByRunningPipelineKey(k); ok {
out = append(out, p)
}
}
return out, nil
}
// Waiting are the waiting.<sequence> keys and the pipeline each names, lowest sequence first.
func (q *Queue) Waiting() ([]string, []string, error) {
ks, err := q.Store.Keys(waitingPrefix)
if err != nil {
return nil, nil, fmt.Errorf("the queue's waiting pipelines are not read: %w", err)
}
slices.Sort(ks)
var keys, pipelines []string
for _, k := range ks {
raw, _, err := q.Store.Get(k)
if errors.Is(err, ErrNoKey) {
continue
} else if err != nil {
return nil, nil, fmt.Errorf("the queue's %s is not read: %w", k, err)
}
w, err := seat.DecodeQueueByWaitingSequenceValue(raw)
if err != nil {
return nil, nil, fmt.Errorf("the queue's %s is no waiting pipeline: %w", k, err)
}
keys, pipelines = append(keys, k), append(pipelines, w.Pipeline)
}
return keys, pipelines, nil
}
// full says whether as many pipelines run as AtOnce allows.
func (q *Queue) full() (bool, error) {
running, err := q.Running()
if err != nil {
return false, err
}
return len(running) >= max(q.AtOnce, 1), nil
}
// run writes key's running.<pipeline> by compare-and-set on its absence; it says whether it wrote it.
func (q *Queue) run(key string) (bool, error) {
raw, err := json.Marshal(seat.QueueByRunningPipelineValue{Pipeline: key})
if err != nil {
return false, err
}
if _, err := q.Store.Create(seat.QueueByRunningPipelineKey(key), raw); errors.Is(err, ErrExists) {
return false, nil
} else if err != nil {
return false, fmt.Errorf("the queue's running key of %s is not written: %w", key, err)
}
return true, nil
}
// next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1.
func (q *Queue) next() (int64, error) {
for {
raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey())
var seq int64 = 1
if err == nil {
if seq, err = seat.DecodeQueueByQueueNextValue(raw); err != nil {
return 0, fmt.Errorf("the queue's counter is no number: %w", err)
}
} else if !errors.Is(err, ErrNoKey) {
return 0, fmt.Errorf("the queue's counter is not read: %w", err)
}
value := []byte(strconv.FormatInt(seq+1, 10))
if errors.Is(err, ErrNoKey) {
_, err = q.Store.Create(seat.QueueByQueueNextKey(), value)
} else {
_, err = q.Store.Update(seat.QueueByQueueNextKey(), value, rev)
}
if errors.Is(err, ErrExists) || errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return 0, fmt.Errorf("the queue's counter is not written: %w", err)
}
return seq, nil
}
}
// wait writes key's waiting.<sequence>.
func (q *Queue) wait(seq int64, key string) error {
raw, err := json.Marshal(seat.QueueByWaitingSequenceValue{Pipeline: key})
if err != nil {
return err
}
if _, err := q.Store.Create(sequenceKey(seq), raw); err != nil {
return fmt.Errorf("the queue's waiting key of %s is not written: %w", key, err)
}
return nil
}
// leave deletes key's running.<pipeline>, a key not written being no error.
func (q *Queue) leave(key string) error {
if err := q.Store.Delete(seat.QueueByRunningPipelineKey(key)); err != nil && !errors.Is(err, ErrNoKey) {
return fmt.Errorf("the queue's running key of %s is not deleted: %w", key, err)
}
return nil
}
// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on <branch>.<source> with step pipeline and
// on pipelines-at-once (design 47 H §748, H-A-52).
func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait {
return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run,
Step: seat.WaitStepPipeline, On: seat.WaitOnPipelinesAtOnce, How: how, Since: at}
}
func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how string) {
if s.Say == nil {
return
}
if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil {
s.logf("%s of %s is not said: %v", event, p.Pipeline, err)
}
}
// queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key
// written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key
// is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A
// start with no queue is placed.
func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) {
if s.Queue == nil {
return true, "", nil
}
full, err := s.Queue.full()
if err != nil || !full {
return err == nil, "", err
}
seq, err := s.Queue.next()
if err != nil {
return false, "", err
}
p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting
return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline)
}
// ran writes the running key of a pipeline placed at its start, after the pipeline is written: one written but not keyed
// (the runner stopped between) runs once over the bound, never holding it.
func (s *Starter) ran(key string) {
if s.Queue == nil {
return
}
if _, err := s.Queue.run(key); err != nil {
s.logf("%s runs, and %v", key, err)
}
}
// StartQueue takes up the queue at the runner's start, from its own state (§748, H-A-14): a running key whose pipeline no
// longer runs (its end heard while the key was not deleted) is deleted, then the waiting are placed (PlaceQueued).
func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
running, err := s.Queue.Running()
if err != nil {
return 0, err
}
for _, key := range running {
if p, _, err := s.Store.Get(key); err == nil && p.Outcome == seat.PipelineOutcomeRunning {
continue
}
s.logf("%s is keyed running in the queue and runs no more: its key is deleted", key)
if err := s.Queue.leave(key); err != nil {
return 0, err
}
}
return s.PlaceQueued(view)
}
// Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the
// queue placed (PlaceQueued).
func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
if err := s.Queue.leave(key); err != nil {
return 0, err
}
return s.PlaceQueued(view)
}
// PlaceQueued places the waiting pipelines, lowest sequence first, while fewer than pipelines-at-once run (design 47 H
// §748): at the runner's start, from its own state, and on every end. Each placed has its running key written and its
// waiting key deleted, becomes running, has its steps placed as a started pipeline's are, and says wait-ended. A waiting key
// whose pipeline is no longer waiting is deleted. It answers how many it placed.
func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error) {
if s.Queue == nil {
return 0, nil
}
placedN := 0
for {
full, err := s.Queue.full()
if err != nil || full {
return placedN, err
}
keys, pipelines, err := s.Queue.Waiting()
if err != nil || len(keys) == 0 {
return placedN, err
}
ok, err := s.placeQueued(keys[0], pipelines[0], view)
if err != nil {
return placedN, err
}
if ok {
placedN++
}
}
}
// placeQueued places the waiting pipeline key, its queue key wkey; false when it was no longer waiting (its key dropped).
func (s *Starter) placeQueued(wkey, key string, view func(branch string) (Half, bool)) (bool, error) {
for {
p, rev, err := s.Store.Get(key)
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting {
if err != nil {
s.logf("the queue's %s names %s, not read (%v): its key is dropped", wkey, key, err)
}
return false, s.dropWaiting(wkey)
}
if _, err := s.Queue.run(key); err != nil {
return false, err
}
now := s.now()
p.Outcome = seat.PipelineOutcomeRunning
if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written running: %w", key, err)
}
if err := s.dropWaiting(wkey); err != nil {
return true, err
}
s.logf("%s is placed: its turn under pipelines-at-once came", key)
s.sayQueueWait(seat.EventWaitEnded, p, now, "ran")
if p.Run != "" {
// A run's steps are placed from its candidate's newest revision, as at the runner's start (A-222).
if s.RunTurn != nil {
s.RunTurn(UnplacedRun{Run: p.Run, Branch: p.Branch})
}
return true, nil
}
if s.Registration != nil && view != nil {
if _, err := s.placeWaiting(s.Store, func(x seat.Pipeline) bool { return x.Pipeline == key }, s.Registration, view, "its turn in the queue"); err != nil {
return true, err
}
}
return true, nil
}
}
func (s *Starter) dropWaiting(wkey string) error {
if err := s.Queue.Store.Delete(wkey); err != nil && !errors.Is(err, ErrNoKey) {
return fmt.Errorf("the queue's %s is not deleted: %w", wkey, err)
}
return nil
}
@@ -0,0 +1,160 @@
package pipeline
import (
"slices"
"strings"
"testing"
"time"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// memQueue is the state queue in memory, written by compare-and-set as the bus's is.
type memQueue map[string]struct {
v []byte
rev uint64
}
func (m memQueue) Get(key string) ([]byte, uint64, error) {
e, ok := m[key]
if !ok {
return nil, 0, ErrNoKey
}
return e.v, e.rev, nil
}
func (m memQueue) Create(key string, v []byte) (uint64, error) {
if _, ok := m[key]; ok {
return 0, ErrExists
}
m[key] = struct {
v []byte
rev uint64
}{v, 1}
return 1, nil
}
func (m memQueue) Update(key string, v []byte, rev uint64) (uint64, error) {
if e, ok := m[key]; !ok || e.rev != rev {
return 0, ErrChanged
}
m[key] = struct {
v []byte
rev uint64
}{v, rev + 1}
return rev + 1, nil
}
func (m memQueue) Delete(key string) error {
if _, ok := m[key]; !ok {
return ErrNoKey
}
delete(m, key)
return nil
}
func (m memQueue) Keys(prefix string) ([]string, error) {
var out []string
for k := range m {
if strings.HasPrefix(k, prefix) {
out = append(out, k)
}
}
slices.Sort(out)
return out, nil
}
// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a
// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started
// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places
// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline
// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed.
func TestThePipelinesWaitTheirTurn(t *testing.T) {
reg, entries := placeFixture()
mem, q := memStore{}, memQueue{}
asks := &stepAsks{}
var said []string
var waits []seat.Wait
say := func(subject string, body any) error {
said = append(said, subject[strings.Index(subject, ".event.")+7:])
if w, ok := body.(seat.Wait); ok {
waits = append(waits, w)
}
return nil
}
now := func() time.Time { return time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC) }
s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: say, Steps: asks, Name: "the-runner", Now: now,
Registration: func(string) (catalog.Registration, bool) { return reg, true },
Queue: &Queue{Store: q, AtOnce: 1}}
view := func(string) (Half, bool) { return Half{Revision: 7, Entries: entries}, true }
h := &Hearer{Store: mem, Name: "the-runner", Say: say, Now: now, Turn: func(key string) { _, _ = s.Turn(key, view) }}
v := Verbs{Hearer: h, Starter: s, Acts: &actLog{}}
take := func(source int64) {
t.Helper()
ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
if n, err := s.TakenIn(ev, view); n != 1 || err != nil {
t.Fatalf("the take-in of source %d started %d (%v)", source, n, err)
}
}
take(4)
first := len(asks.ids)
if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 {
t.Fatalf("the first pipeline = %s seq %d with %d asks, want running and asked", p.Outcome, p.Seq, first)
}
if _, ok := q["running.main.4"]; !ok {
t.Fatalf("the queue = %v, want running.main.4", q)
}
said, waits = nil, nil
take(5)
take(6)
for i, key := range []string{"main.5", "main.6"} {
p, _, _ := mem.Get(key)
if p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != int64(i+1) || len(p.Steps) != 0 {
t.Errorf("%s = %s seq %d with %d steps, want waiting at %d with none", key, p.Outcome, p.Seq, len(p.Steps), i+1)
}
}
if len(asks.ids) != first {
t.Errorf("a waiting pipeline asked %d steps", len(asks.ids)-first)
}
if _, ok := q["waiting.00000000000000000001"]; !ok || string(q["queue.next"].v) != "3" {
t.Errorf("the queue = %v, want waiting.<1 in 20 digits> and queue.next 3", q)
}
if !slices.Contains(said, "pipeline-started.main.5") || !slices.Contains(said, "wait-held.main.5") || len(waits) != 2 ||
waits[0].Step != seat.WaitStepPipeline || waits[0].On != seat.WaitOnPipelinesAtOnce || waits[0].Pipeline != "main.5" {
t.Errorf("said %v, waits %+v; want pipeline-started and wait-held step pipeline on pipelines-at-once", said, waits)
}
// R11/AC2: a cancel places the next, the lowest sequence.
said, waits = nil, nil
if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: "main.4", Why: "wrong"}); err != nil {
t.Fatal(err)
}
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(p.Steps) == 0 || len(asks.ids) == first {
t.Errorf("after the cancel main.5 = %s with %d steps, want placed and asked", p.Outcome, len(p.Steps))
}
if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeWaiting {
t.Errorf("main.6 = %s, want still waiting", p.Outcome)
}
_, gone := q["running.main.4"]
_, runs := q["running.main.5"]
_, waits5 := q["waiting.00000000000000000001"]
if gone || !runs || waits5 {
t.Errorf("the queue after the cancel = %v", q)
}
if !slices.Contains(said, "wait-ended.main.5") || len(waits) != 1 || waits[0].How != "ran" {
t.Errorf("said %v, waits %+v; want wait-ended of main.5", said, waits)
}
// At the start: main.5's end was written while its key stayed, and a waiting key names a pipeline never written.
p5, rev, _ := mem.Get("main.5")
p5.Outcome = seat.PipelineOutcomePassed
mem.Update("main.5", p5, rev)
q.Create("waiting.00000000000000000000", []byte(`{"pipeline":"main.0"}`))
if n, err := s.StartQueue(view); n != 1 || err != nil {
t.Fatalf("the start placed %d (%v), want main.6", n, err)
}
_, stale := q["running.main.5"]
_, lost := q["waiting.00000000000000000000"]
if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning || stale || lost {
t.Errorf("after the start main.6 = %s, queue %v", p.Outcome, q)
}
}
@@ -79,9 +79,24 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err
key := RunKey(branch, id)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: branch, Source: source, Movers: movers, Run: id,
Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}}
// A run waits in the same queue as every pipeline (§748, H-A-52): over pipelines-at-once it is written waiting, and its
// steps are placed from its candidate's newest revision at its turn (RunTurn).
place, wkey, err := s.queueStart(&p)
if err != nil {
return seat.RunAnswer{}, fmt.Errorf("the run's pipeline %s is not judged against pipelines-at-once: %w", key, err)
}
if err := s.create(key, p); err != nil {
if wkey != "" {
_ = s.dropWaiting(wkey)
}
return seat.RunAnswer{}, fmt.Errorf("the pipeline %s of the run is not written: %w", key, err)
}
if place {
s.ran(key)
} else {
s.logf("the run %s waits its turn under pipelines-at-once, sequence %d", id, p.Seq)
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
}
first := in.Members[0]
last := first.Commit
if len(in.Members) > 1 {
@@ -170,6 +170,12 @@ type Starter struct {
Runs RunSettings
RunsFill func() (seat.Fill, error)
EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error
// Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil
// places every start at once.
Queue *Queue
// RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places
// it at the runner's next start.
RunTurn func(UnplacedRun)
}
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
@@ -418,7 +424,13 @@ func (s *Starter) Start(b Begin) (bool, error) {
p := newPipeline(b, key, now)
var asks []Asked
var ended, notAsked []int
if b.unheld {
place, wkey, err := s.queueStart(&p)
if err != nil {
return false, fmt.Errorf("the pipeline %s is not judged against pipelines-at-once: %w", key, err)
}
if !place {
s.logf("%s waits its turn under pipelines-at-once, sequence %d", key, p.Seq)
} else if b.unheld {
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
} else if s.Registration != nil {
var unregistered []string
@@ -427,13 +439,24 @@ func (s *Starter) Start(b Begin) (bool, error) {
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
}
}
if err := s.create(key, p); errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
} else if err != nil {
if err := s.create(key, p); err != nil {
if wkey != "" {
_ = s.dropWaiting(wkey)
}
if errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
}
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
// A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its
// wait on its turn.
s.sayStartedPipeline(p, now)
if !place {
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
return true, nil
}
s.ran(key)
s.sayNotAsked(p, notAsked)
if len(ended) > 0 {
return true, s.endedAtPlacing(p, ended)
@@ -155,6 +155,9 @@ type Hearer struct {
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
// edge, §746; Starter.Release); nil places none.
Release func(key string)
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
// Starter.Turn); nil takes none.
Turn func(key string)
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
@@ -413,7 +416,12 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
}
if ended {
h.Failures.Ended(p)
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
err := h.say(seat.EventPipelineEnded, p, nil)
// Every end, a cancel's alike, places the next waiting (§748, H-A-53), said or not.
if h.Turn != nil {
h.Turn(p.Pipeline)
}
if err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
}
@@ -55,11 +55,13 @@ func TestManifest(t *testing.T) {
{"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat &&
equalSets(m.Claims[0].Serves, seat.Verbs())},
// §27, §85, §330, §355, §77: the old state sections.
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})},
{"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue})},
// §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live.
{"bounds is key-ttl", keyTTL["bounds"]},
// §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling.
{"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64},
// §748: the settings of the test runs, the widens and the queue, pipelines-at-once from 1 to 8 (default 1) and
// pipelines-waiting-cap (default 64), each written to its file.
{"the test runs' settings", runSettingsOf(raw)},
// RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB.
{"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256},
@@ -197,8 +199,11 @@ func runSettingsOf(raw []byte) bool {
if json.Unmarshal(raw, &m) != nil {
return false
}
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) {
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) {
return false
}
env := map[string]bool{}
+38 -1
View File
@@ -127,6 +127,21 @@
"trusted": false,
"default": 6
},
"pipelines-at-once": {
"type": "number",
"description": "how many pipelines have their steps asked at once; a pipeline started over it waits its turn in the state queue, placed in order on every end (design 47 H section 748).",
"minimum": 1,
"maximum": 8,
"trusted": false,
"default": 1
},
"pipelines-waiting-cap": {
"type": "number",
"description": "how many pipelines may wait their turn under pipelines-at-once; a start over it raises the finding queue-full and is taken on the next end (design 47 H section 748).",
"minimum": 1,
"trusted": false,
"default": 64
},
"run-untaken-bound": {
"type": "string",
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
@@ -145,6 +160,10 @@
"history": 1,
"max-mib": 128
},
{
"name": "queue",
"history": 1
},
{
"name": "aged",
"history": 1
@@ -271,6 +290,22 @@
"mode": "0600",
"content": "${config:revisions-kept}"
},
{
"id": "pipelines-at-once",
"type": "file",
"path": "${dir:state}/pipelines-at-once",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:pipelines-at-once}"
},
{
"id": "pipelines-waiting-cap",
"type": "file",
"path": "${dir:state}/pipelines-waiting-cap",
"owner": "mesh-delivery-pipeline",
"mode": "0600",
"content": "${config:pipelines-waiting-cap}"
},
{
"id": "pipeline",
"type": "process",
@@ -286,7 +321,9 @@
"MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound",
"MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound",
"MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept",
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound"
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
"MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once",
"MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap"
},
"user": "mesh-delivery-pipeline",
"health": {