diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 76a234ad..479018dd 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -196,6 +196,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } registrar.Store(®ister) starter.Store = pipelineStore{ctx: ctx, kv: pipelines} + // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in + // the state queue, placed at the start from it and on every end. + atOnce, waitingCap, err := queueSettings(getenv) + if err != nil { + return err + } + queueBucket, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, seat.StateQueue), "keep the pipelines waiting their turn") + if err != nil { + return err + } + logger.Printf("the queue: %d pipelines at once, at most %d waiting", atOnce, waitingCap) + starter.Queue = &pipeline.Queue{Store: queueKV{ctx: ctx, kv: queueBucket}, AtOnce: atOnce} + hearer.Turn = func(key string) { + if n, err := starter.Turn(key, rv.Catalog.View); err != nil || n > 0 { + logger.Printf("the end of %s placed %d waiting pipelines (%v)", key, n, err) + } + } + starter.RunTurn = func(u pipeline.UnplacedRun) { + one := func() ([]pipeline.UnplacedRun, error) { return []pipeline.UnplacedRun{u}, nil } + if err := placeLostRuns(one, calls.CallKept, placeHalf, logger.Printf); err != nil { + logger.Printf("the run %s whose turn came is not placed: %v", u.Run, err) + } + } // A run's placing lost while the runner was down is taken up from its candidate's newest revision, once the registrations // that place its steps are held (A-222). go func() { @@ -212,6 +235,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := placeLostRuns(starter.UnplacedRuns, calls.CallKept, placeHalf, logger.Printf); err != nil { logger.Printf("the runs not placed at the start: %v", err) } + // The queue is placed at the start from its own state (§748). + if n, err := starter.StartQueue(rv.Catalog.View); err != nil || n > 0 { + logger.Printf("the queue at the start: %d waiting pipelines placed (%v)", n, err) + } }() // A run's candidate revision places its pipelines' steps, its entries read through the seat's revision verb (A-180). place := func(ev catalog.CatalogRevisionEvent) error { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue.go new file mode 100644 index 00000000..f131f2b3 --- /dev/null +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-pipeline/internal/pipeline" +) + +const ( + // envPipelinesAtOnce and envPipelinesWaitingCap name the files of the settings pipelines-at-once and + // pipelines-waiting-cap (design 47 H §748). + envPipelinesAtOnce = "MESH_RUNNER_PIPELINES_AT_ONCE_FILE" + envPipelinesWaitingCap = "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE" + defaultPipelinesAtOnce = 1 + maxPipelinesAtOnce = 8 + defaultWaitingCap = 64 +) + +// queueSettings reads pipelines-at-once (default 1, from 1 to 8) and pipelines-waiting-cap (default 64) from their files +// (design 47 H §748): a file missing or empty is the default, any other value outside its range an error naming the file. +func queueSettings(getenv func(string) string) (int, int, error) { + read := func(env string, def, lo, hi int) (int, error) { + v, err := settingText(getenv(env)) + if err != nil || v == "" { + return def, err + } + n, err := strconv.Atoi(v) + if err != nil || n < lo || (hi > 0 && n > hi) { + return 0, fmt.Errorf("%s holds %q, which is not a whole number in range (at least %d, at most %d where above 0)", getenv(env), v, lo, hi) + } + return n, nil + } + atOnce, err := read(envPipelinesAtOnce, defaultPipelinesAtOnce, 1, maxPipelinesAtOnce) + if err != nil { + return 0, 0, err + } + waitingCap, err := read(envPipelinesWaitingCap, defaultWaitingCap, 1, 0) + return atOnce, waitingCap, err +} + +// queueKV is the runner's state queue on the bus, written by compare-and-set (design 47 H §748). +type queueKV struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (q queueKV) Get(key string) ([]byte, uint64, error) { + e, err := q.kv.Get(q.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, pipeline.ErrNoKey + } else if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (q queueKV) Create(key string, value []byte) (uint64, error) { + rev, err := q.kv.Create(q.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, pipeline.ErrExists + } + return rev, err +} + +func (q queueKV) Update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := q.kv.Update(q.ctx, key, value, revision) + if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { + return 0, pipeline.ErrChanged + } + return rev, err +} + +func (q queueKV) Delete(key string) error { + err := q.kv.Delete(q.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return pipeline.ErrNoKey + } + return err +} + +func (q queueKV) Keys(prefix string) ([]string, error) { + lister, err := q.kv.ListKeysFiltered(q.ctx, prefix+">") + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return nil, err + } + var out []string + for k := range lister.Keys() { + out = append(out, k) + } + slices.Sort(out) + return out, nil +} diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 807c8c0f..1765d80e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -60,7 +60,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes defer cancel() const module = "mesh-delivery-pipeline" - for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} { + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go index c40874b0..95ed9670 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/testruns_integration_test.go @@ -58,7 +58,7 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { defer cancel() const module = "mesh-delivery-pipeline" - for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries"} { + for _, state := range []string{"pipelines", "kept", "toolchains", "findings", "first-views", "consumers", "aged", "recoveries", "queue"} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: module + "_" + state, History: 1}); err != nil { t.Fatal(err) } @@ -186,9 +186,10 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) { return out } - // The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds. + // The settings, as the manifest's resources write them (A-469x): a cap of 1 MiB, a step bound of two seconds; the + // candidates' pipelines run side by side, so pipelines-at-once is its most, 8 (§748). dir := t.TempDir() - files := map[string]string{envRunsCap: "1", envStepBound: "2s"} + files := map[string]string{envRunsCap: "1", envStepBound: "2s", envPipelinesAtOnce: "8"} env := map[string]string{} for name, content := range files { path := filepath.Join(dir, name) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens_test.go index 07abf252..57c7fea7 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/widens_test.go @@ -131,4 +131,28 @@ func TestWidensSettings(t *testing.T) { } }) } + // R11 (design 47 H §748): pipelines-at-once, default 1, from 1 to 8; pipelines-waiting-cap, default 64. + for _, c := range []struct { + name string + files map[string]string + atOnce, limit int + ok bool + }{ + {"queue defaults", nil, 1, 64, true}, + {"queue set", map[string]string{envPipelinesAtOnce: "3\n", envPipelinesWaitingCap: "10"}, 3, 10, true}, + {"over 8 at once", map[string]string{envPipelinesAtOnce: "9"}, 0, 0, false}, + {"none at once", map[string]string{envPipelinesAtOnce: "0"}, 0, 0, false}, + {"no cap", map[string]string{envPipelinesWaitingCap: "0"}, 1, 0, false}, + } { + t.Run(c.name, func(t *testing.T) { + env := map[string]string{} + for k, v := range c.files { + env[k] = write(c.name+k, v) + } + atOnce, limit, err := queueSettings(func(k string) string { return env[k] }) + if (err == nil) != c.ok || (c.ok && (atOnce != c.atOnce || limit != c.limit)) { + t.Fatalf("%d %d, %v", atOnce, limit, err) + } + }) + } } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go new file mode 100644 index 00000000..86755f00 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -0,0 +1,312 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "time" + + runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// ErrNoKey is a QueueStore's answer to a Get of a key it does not hold. +var ErrNoKey = errors.New("the queue holds no such key") + +// QueueStore is the runner's state queue (design 47 H §748, H-A-36, H-A-72 (2), H-A-96): its counter queue.next, one key +// waiting. per waiting pipeline and one key running. per pipeline whose steps are asked. A write is by +// compare-and-set: Create on a key's absence (ErrExists when it is written), Update on the revision read (ErrChanged). +type QueueStore interface { + Get(key string) ([]byte, uint64, error) + Create(key string, value []byte) (uint64, error) + Update(key string, value []byte, revision uint64) (uint64, error) + Delete(key string) error + // Keys are the keys beginning with prefix, in key order. + Keys(prefix string) ([]string, error) +} + +// Queue bounds the pipelines whose steps are asked at once to AtOnce, the setting pipelines-at-once (design 47 H §748, +// H-A-14): it is judged against the running. keys alone (H-A-96); a pipeline over it waits under the next value +// of queue.next. +type Queue struct { + Store QueueStore + AtOnce int +} + +const ( + runningPrefix = "running." + waitingPrefix = "waiting." +) + +// sequenceKey is a waiting pipeline's key, its sequence 20 digits, so the lowest is read first by key order (H-A-72 (2)). +func sequenceKey(seq int64) string { + return seat.QueueByWaitingSequenceKey(fmt.Sprintf("%020d", seq)) +} + +// Running are the pipelines whose running. key is written. +func (q *Queue) Running() ([]string, error) { + ks, err := q.Store.Keys(runningPrefix) + if err != nil { + return nil, fmt.Errorf("the queue's running pipelines are not read: %w", err) + } + out := make([]string, 0, len(ks)) + for _, k := range ks { + if p, ok := seat.ParseQueueByRunningPipelineKey(k); ok { + out = append(out, p) + } + } + return out, nil +} + +// Waiting are the waiting. keys and the pipeline each names, lowest sequence first. +func (q *Queue) Waiting() ([]string, []string, error) { + ks, err := q.Store.Keys(waitingPrefix) + if err != nil { + return nil, nil, fmt.Errorf("the queue's waiting pipelines are not read: %w", err) + } + slices.Sort(ks) + var keys, pipelines []string + for _, k := range ks { + raw, _, err := q.Store.Get(k) + if errors.Is(err, ErrNoKey) { + continue + } else if err != nil { + return nil, nil, fmt.Errorf("the queue's %s is not read: %w", k, err) + } + w, err := seat.DecodeQueueByWaitingSequenceValue(raw) + if err != nil { + return nil, nil, fmt.Errorf("the queue's %s is no waiting pipeline: %w", k, err) + } + keys, pipelines = append(keys, k), append(pipelines, w.Pipeline) + } + return keys, pipelines, nil +} + +// full says whether as many pipelines run as AtOnce allows. +func (q *Queue) full() (bool, error) { + running, err := q.Running() + if err != nil { + return false, err + } + return len(running) >= max(q.AtOnce, 1), nil +} + +// run writes key's running. by compare-and-set on its absence; it says whether it wrote it. +func (q *Queue) run(key string) (bool, error) { + raw, err := json.Marshal(seat.QueueByRunningPipelineValue{Pipeline: key}) + if err != nil { + return false, err + } + if _, err := q.Store.Create(seat.QueueByRunningPipelineKey(key), raw); errors.Is(err, ErrExists) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("the queue's running key of %s is not written: %w", key, err) + } + return true, nil +} + +// next is the sequence the next waiting pipeline gets: queue.next read and written one higher by compare-and-set, from 1. +func (q *Queue) next() (int64, error) { + for { + raw, rev, err := q.Store.Get(seat.QueueByQueueNextKey()) + var seq int64 = 1 + if err == nil { + if seq, err = seat.DecodeQueueByQueueNextValue(raw); err != nil { + return 0, fmt.Errorf("the queue's counter is no number: %w", err) + } + } else if !errors.Is(err, ErrNoKey) { + return 0, fmt.Errorf("the queue's counter is not read: %w", err) + } + value := []byte(strconv.FormatInt(seq+1, 10)) + if errors.Is(err, ErrNoKey) { + _, err = q.Store.Create(seat.QueueByQueueNextKey(), value) + } else { + _, err = q.Store.Update(seat.QueueByQueueNextKey(), value, rev) + } + if errors.Is(err, ErrExists) || errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return 0, fmt.Errorf("the queue's counter is not written: %w", err) + } + return seq, nil + } +} + +// wait writes key's waiting.. +func (q *Queue) wait(seq int64, key string) error { + raw, err := json.Marshal(seat.QueueByWaitingSequenceValue{Pipeline: key}) + if err != nil { + return err + } + if _, err := q.Store.Create(sequenceKey(seq), raw); err != nil { + return fmt.Errorf("the queue's waiting key of %s is not written: %w", key, err) + } + return nil +} + +// leave deletes key's running., a key not written being no error. +func (q *Queue) leave(key string) error { + if err := q.Store.Delete(seat.QueueByRunningPipelineKey(key)); err != nil && !errors.Is(err, ErrNoKey) { + return fmt.Errorf("the queue's running key of %s is not deleted: %w", key, err) + } + return nil +} + +// queueWait is a pipeline's wait on its turn, said as wait-held and wait-ended on . with step pipeline and +// on pipelines-at-once (design 47 H §748, H-A-52). +func queueWait(p seat.Pipeline, at time.Time, how string) seat.Wait { + return seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Revision: p.Revision, Run: p.Run, + Step: seat.WaitStepPipeline, On: seat.WaitOnPipelinesAtOnce, How: how, Since: at} +} + +func (s *Starter) sayQueueWait(event string, p seat.Pipeline, at time.Time, how string) { + if s.Say == nil { + return + } + if err := s.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), queueWait(p, at, how)); err != nil { + s.logf("%s of %s is not said: %v", event, p.Pipeline, err) + } +} + +// queueStart judges a start against the bound (§748): under it, it answers true, the pipeline placed and its running key +// written once the pipeline is (ran); over it p gets the next sequence and state waiting, with no step, and its waiting key +// is written first, so a key whose pipeline was never written is dropped at its placing; it answers false and that key. A +// start with no queue is placed. +func (s *Starter) queueStart(p *seat.Pipeline) (bool, string, error) { + if s.Queue == nil { + return true, "", nil + } + full, err := s.Queue.full() + if err != nil || !full { + return err == nil, "", err + } + seq, err := s.Queue.next() + if err != nil { + return false, "", err + } + p.Seq, p.Outcome = seq, seat.PipelineOutcomeWaiting + return false, sequenceKey(seq), s.Queue.wait(seq, p.Pipeline) +} + +// ran writes the running key of a pipeline placed at its start, after the pipeline is written: one written but not keyed +// (the runner stopped between) runs once over the bound, never holding it. +func (s *Starter) ran(key string) { + if s.Queue == nil { + return + } + if _, err := s.Queue.run(key); err != nil { + s.logf("%s runs, and %v", key, err) + } +} + +// StartQueue takes up the queue at the runner's start, from its own state (§748, H-A-14): a running key whose pipeline no +// longer runs (its end heard while the key was not deleted) is deleted, then the waiting are placed (PlaceQueued). +func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) { + if s.Queue == nil { + return 0, nil + } + running, err := s.Queue.Running() + if err != nil { + return 0, err + } + for _, key := range running { + if p, _, err := s.Store.Get(key); err == nil && p.Outcome == seat.PipelineOutcomeRunning { + continue + } + s.logf("%s is keyed running in the queue and runs no more: its key is deleted", key) + if err := s.Queue.leave(key); err != nil { + return 0, err + } + } + return s.PlaceQueued(view) +} + +// Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the +// queue placed (PlaceQueued). +func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) { + if s.Queue == nil { + return 0, nil + } + if err := s.Queue.leave(key); err != nil { + return 0, err + } + return s.PlaceQueued(view) +} + +// PlaceQueued places the waiting pipelines, lowest sequence first, while fewer than pipelines-at-once run (design 47 H +// §748): at the runner's start, from its own state, and on every end. Each placed has its running key written and its +// waiting key deleted, becomes running, has its steps placed as a started pipeline's are, and says wait-ended. A waiting key +// whose pipeline is no longer waiting is deleted. It answers how many it placed. +func (s *Starter) PlaceQueued(view func(branch string) (Half, bool)) (int, error) { + if s.Queue == nil { + return 0, nil + } + placedN := 0 + for { + full, err := s.Queue.full() + if err != nil || full { + return placedN, err + } + keys, pipelines, err := s.Queue.Waiting() + if err != nil || len(keys) == 0 { + return placedN, err + } + ok, err := s.placeQueued(keys[0], pipelines[0], view) + if err != nil { + return placedN, err + } + if ok { + placedN++ + } + } +} + +// placeQueued places the waiting pipeline key, its queue key wkey; false when it was no longer waiting (its key dropped). +func (s *Starter) placeQueued(wkey, key string, view func(branch string) (Half, bool)) (bool, error) { + for { + p, rev, err := s.Store.Get(key) + if err != nil || p.Outcome != seat.PipelineOutcomeWaiting { + if err != nil { + s.logf("the queue's %s names %s, not read (%v): its key is dropped", wkey, key, err) + } + return false, s.dropWaiting(wkey) + } + if _, err := s.Queue.run(key); err != nil { + return false, err + } + now := s.now() + p.Outcome = seat.PipelineOutcomeRunning + if _, err := s.Store.Update(key, p, rev); errors.Is(err, ErrChanged) { + continue + } else if err != nil { + return false, fmt.Errorf("the pipeline %s is not written running: %w", key, err) + } + if err := s.dropWaiting(wkey); err != nil { + return true, err + } + s.logf("%s is placed: its turn under pipelines-at-once came", key) + s.sayQueueWait(seat.EventWaitEnded, p, now, "ran") + if p.Run != "" { + // A run's steps are placed from its candidate's newest revision, as at the runner's start (A-222). + if s.RunTurn != nil { + s.RunTurn(UnplacedRun{Run: p.Run, Branch: p.Branch}) + } + return true, nil + } + if s.Registration != nil && view != nil { + if _, err := s.placeWaiting(s.Store, func(x seat.Pipeline) bool { return x.Pipeline == key }, s.Registration, view, "its turn in the queue"); err != nil { + return true, err + } + } + return true, nil + } +} + +func (s *Starter) dropWaiting(wkey string) error { + if err := s.Queue.Store.Delete(wkey); err != nil && !errors.Is(err, ErrNoKey) { + return fmt.Errorf("the queue's %s is not deleted: %w", wkey, err) + } + return nil +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go new file mode 100644 index 00000000..5ca3f608 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue_test.go @@ -0,0 +1,160 @@ +package pipeline + +import ( + "slices" + "strings" + "testing" + "time" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// memQueue is the state queue in memory, written by compare-and-set as the bus's is. +type memQueue map[string]struct { + v []byte + rev uint64 +} + +func (m memQueue) Get(key string) ([]byte, uint64, error) { + e, ok := m[key] + if !ok { + return nil, 0, ErrNoKey + } + return e.v, e.rev, nil +} + +func (m memQueue) Create(key string, v []byte) (uint64, error) { + if _, ok := m[key]; ok { + return 0, ErrExists + } + m[key] = struct { + v []byte + rev uint64 + }{v, 1} + return 1, nil +} + +func (m memQueue) Update(key string, v []byte, rev uint64) (uint64, error) { + if e, ok := m[key]; !ok || e.rev != rev { + return 0, ErrChanged + } + m[key] = struct { + v []byte + rev uint64 + }{v, rev + 1} + return rev + 1, nil +} + +func (m memQueue) Delete(key string) error { + if _, ok := m[key]; !ok { + return ErrNoKey + } + delete(m, key) + return nil +} + +func (m memQueue) Keys(prefix string) ([]string, error) { + var out []string + for k := range m { + if strings.HasPrefix(k, prefix) { + out = append(out, k) + } + } + slices.Sort(out) + return out, nil +} + +// R11/AC1, R11/AC2 (design 47 H §748, H-A-14, H-A-36, H-A-52, H-A-72, H-A-96): with pipelines-at-once 1, a second and a +// third take-in are written waiting under the next sequences of queue.next, with no step asked, each saying pipeline-started +// and wait-held step pipeline on pipelines-at-once; the first's end, here its cancel, deletes its running key and places +// the lowest sequence, which says wait-ended and has its steps asked; at the runner's start a running key whose pipeline +// runs no more is deleted, a waiting key whose pipeline is gone dropped, and the next waiting placed. +func TestThePipelinesWaitTheirTurn(t *testing.T) { + reg, entries := placeFixture() + mem, q := memStore{}, memQueue{} + asks := &stepAsks{} + var said []string + var waits []seat.Wait + say := func(subject string, body any) error { + said = append(said, subject[strings.Index(subject, ".event.")+7:]) + if w, ok := body.(seat.Wait); ok { + waits = append(waits, w) + } + return nil + } + now := func() time.Time { return time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC) } + s := &Starter{Pipelines: memCreated{mem}, Store: mem, Say: say, Steps: asks, Name: "the-runner", Now: now, + Registration: func(string) (catalog.Registration, bool) { return reg, true }, + Queue: &Queue{Store: q, AtOnce: 1}} + view := func(string) (Half, bool) { return Half{Revision: 7, Entries: entries}, true } + h := &Hearer{Store: mem, Name: "the-runner", Say: say, Now: now, Turn: func(key string) { _, _ = s.Turn(key, view) }} + v := Verbs{Hearer: h, Starter: s, Acts: &actLog{}} + take := func(source int64) { + t.Helper() + ev := catalog.TakeInTaken{Branch: "main", Source: source, Call: "c0ffee", Revision: 7, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}} + if n, err := s.TakenIn(ev, view); n != 1 || err != nil { + t.Fatalf("the take-in of source %d started %d (%v)", source, n, err) + } + } + take(4) + first := len(asks.ids) + if p, _, _ := mem.Get("main.4"); p.Outcome != seat.PipelineOutcomeRunning || p.Seq != 0 || first == 0 { + t.Fatalf("the first pipeline = %s seq %d with %d asks, want running and asked", p.Outcome, p.Seq, first) + } + if _, ok := q["running.main.4"]; !ok { + t.Fatalf("the queue = %v, want running.main.4", q) + } + said, waits = nil, nil + take(5) + take(6) + for i, key := range []string{"main.5", "main.6"} { + p, _, _ := mem.Get(key) + if p.Outcome != seat.PipelineOutcomeWaiting || p.Seq != int64(i+1) || len(p.Steps) != 0 { + t.Errorf("%s = %s seq %d with %d steps, want waiting at %d with none", key, p.Outcome, p.Seq, len(p.Steps), i+1) + } + } + if len(asks.ids) != first { + t.Errorf("a waiting pipeline asked %d steps", len(asks.ids)-first) + } + if _, ok := q["waiting.00000000000000000001"]; !ok || string(q["queue.next"].v) != "3" { + t.Errorf("the queue = %v, want waiting.<1 in 20 digits> and queue.next 3", q) + } + if !slices.Contains(said, "pipeline-started.main.5") || !slices.Contains(said, "wait-held.main.5") || len(waits) != 2 || + waits[0].Step != seat.WaitStepPipeline || waits[0].On != seat.WaitOnPipelinesAtOnce || waits[0].Pipeline != "main.5" { + t.Errorf("said %v, waits %+v; want pipeline-started and wait-held step pipeline on pipelines-at-once", said, waits) + } + // R11/AC2: a cancel places the next, the lowest sequence. + said, waits = nil, nil + if _, err := v.Cancel("operator", seat.CancelInput{Pipeline: "main.4", Why: "wrong"}); err != nil { + t.Fatal(err) + } + if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(p.Steps) == 0 || len(asks.ids) == first { + t.Errorf("after the cancel main.5 = %s with %d steps, want placed and asked", p.Outcome, len(p.Steps)) + } + if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeWaiting { + t.Errorf("main.6 = %s, want still waiting", p.Outcome) + } + _, gone := q["running.main.4"] + _, runs := q["running.main.5"] + _, waits5 := q["waiting.00000000000000000001"] + if gone || !runs || waits5 { + t.Errorf("the queue after the cancel = %v", q) + } + if !slices.Contains(said, "wait-ended.main.5") || len(waits) != 1 || waits[0].How != "ran" { + t.Errorf("said %v, waits %+v; want wait-ended of main.5", said, waits) + } + // At the start: main.5's end was written while its key stayed, and a waiting key names a pipeline never written. + p5, rev, _ := mem.Get("main.5") + p5.Outcome = seat.PipelineOutcomePassed + mem.Update("main.5", p5, rev) + q.Create("waiting.00000000000000000000", []byte(`{"pipeline":"main.0"}`)) + if n, err := s.StartQueue(view); n != 1 || err != nil { + t.Fatalf("the start placed %d (%v), want main.6", n, err) + } + _, stale := q["running.main.5"] + _, lost := q["waiting.00000000000000000000"] + if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning || stale || lost { + t.Errorf("after the start main.6 = %s, queue %v", p.Outcome, q) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/run.go b/modules/mesh-delivery-pipeline/internal/pipeline/run.go index 02926744..987e44d9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/run.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/run.go @@ -79,9 +79,24 @@ func (s *Starter) Run(caller string, in seat.RunInput, newID func() (string, err key := RunKey(branch, id) p := seat.Pipeline{V: 1, Pipeline: key, Branch: branch, Source: source, Movers: movers, Run: id, Outcome: seat.PipelineOutcomeRunning, Started: &now, Steps: []seat.StepResult{}} + // A run waits in the same queue as every pipeline (§748, H-A-52): over pipelines-at-once it is written waiting, and its + // steps are placed from its candidate's newest revision at its turn (RunTurn). + place, wkey, err := s.queueStart(&p) + if err != nil { + return seat.RunAnswer{}, fmt.Errorf("the run's pipeline %s is not judged against pipelines-at-once: %w", key, err) + } if err := s.create(key, p); err != nil { + if wkey != "" { + _ = s.dropWaiting(wkey) + } return seat.RunAnswer{}, fmt.Errorf("the pipeline %s of the run is not written: %w", key, err) } + if place { + s.ran(key) + } else { + s.logf("the run %s waits its turn under pipelines-at-once, sequence %d", id, p.Seq) + s.sayQueueWait(seat.EventWaitHeld, p, now, "") + } first := in.Members[0] last := first.Commit if len(in.Members) > 1 { diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 97db1618..63512835 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -170,6 +170,12 @@ type Starter struct { Runs RunSettings RunsFill func() (seat.Fill, error) EndReaders func(run string, readers []seat.TestRunReader, r seat.TestRun) error + // Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil + // places every start at once. + Queue *Queue + // RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places + // it at the runner's next start. + RunTurn func(UnplacedRun) } // holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view @@ -418,7 +424,13 @@ func (s *Starter) Start(b Begin) (bool, error) { p := newPipeline(b, key, now) var asks []Asked var ended, notAsked []int - if b.unheld { + place, wkey, err := s.queueStart(&p) + if err != nil { + return false, fmt.Errorf("the pipeline %s is not judged against pipelines-at-once: %w", key, err) + } + if !place { + s.logf("%s waits its turn under pipelines-at-once, sequence %d", key, p.Seq) + } else if b.unheld { s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch) } else if s.Registration != nil { var unregistered []string @@ -427,13 +439,24 @@ func (s *Starter) Start(b Begin) (bool, error) { s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m) } } - if err := s.create(key, p); errors.Is(err, ErrExists) { - s.logf("%s is started already: this start writes nothing", key) - return false, nil - } else if err != nil { + if err := s.create(key, p); err != nil { + if wkey != "" { + _ = s.dropWaiting(wkey) + } + if errors.Is(err, ErrExists) { + s.logf("%s is started already: this start writes nothing", key) + return false, nil + } return false, fmt.Errorf("the pipeline %s is not written: %w", key, err) } + // A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its + // wait on its turn. s.sayStartedPipeline(p, now) + if !place { + s.sayQueueWait(seat.EventWaitHeld, p, now, "") + return true, nil + } + s.ran(key) s.sayNotAsked(p, notAsked) if len(ended) > 0 { return true, s.endedAtPlacing(p, ended) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index 58be04e4..96c4cd8e 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -155,6 +155,9 @@ type Hearer struct { // Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3 // edge, §746; Starter.Release); nil places none. Release func(key string) + // Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748; + // Starter.Turn); nil takes none. + Turn func(key string) } // EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps @@ -413,7 +416,12 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { } if ended { h.Failures.Ended(p) - if err := h.say(seat.EventPipelineEnded, p, nil); err != nil { + err := h.say(seat.EventPipelineEnded, p, nil) + // Every end, a cancel's alike, places the next waiting (§748, H-A-53), said or not. + if h.Turn != nil { + h.Turn(p.Pipeline) + } + if err != nil { return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err) } } diff --git a/modules/mesh-delivery-pipeline/manifest_test.go b/modules/mesh-delivery-pipeline/manifest_test.go index f55cb743..6554237a 100644 --- a/modules/mesh-delivery-pipeline/manifest_test.go +++ b/modules/mesh-delivery-pipeline/manifest_test.go @@ -55,11 +55,13 @@ func TestManifest(t *testing.T) { {"claims the seat serving every verb of its definition", len(m.Claims) == 1 && m.Claims[0].Name == seat.Seat && equalSets(m.Claims[0].Serves, seat.Verbs())}, // §27, §85, §330, §355, §77: the old state sections. - {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers"})}, + {"old state sections", equalSets(states, []string{seat.StatePipelines, seat.StateKept, "aged", "recoveries", "bounds", "test-runs", "toolchains", "findings", "first-views", "consumers", seat.StateQueue})}, // §535, §647, A-77, A-104: an ask no holder takes is bound by a key with its own time to live. {"bounds is key-ttl", keyTTL["bounds"]}, // §725, A-469w, A-470: test-runs holds its runs' bound keys, each with its own time to live; 64 MiB its ceiling. {"test-runs is key-ttl of 64 MiB", keyTTL["test-runs"] && stateMiB["test-runs"] == 64}, + // §748: the settings of the test runs, the widens and the queue, pipelines-at-once from 1 to 8 (default 1) and + // pipelines-waiting-cap (default 64), each written to its file. {"the test runs' settings", runSettingsOf(raw)}, // RN25 (§469, §85): an old state section declares its cap, pipelines 256 MiB and kept 128 MiB. {"pipelines 256 MiB", stateMiB[seat.StatePipelines] == 256}, @@ -197,8 +199,11 @@ func runSettingsOf(raw []byte) bool { if json.Unmarshal(raw, &m) != nil { return false } - want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number"} - if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) { + want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number", + "pipelines-at-once": "number", "pipelines-waiting-cap": "number"} + if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) || + m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) || + m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) { return false } env := map[string]bool{} diff --git a/modules/mesh-delivery-pipeline/module.json b/modules/mesh-delivery-pipeline/module.json index 0fcd5898..5f5a7639 100644 --- a/modules/mesh-delivery-pipeline/module.json +++ b/modules/mesh-delivery-pipeline/module.json @@ -127,6 +127,21 @@ "trusted": false, "default": 6 }, + "pipelines-at-once": { + "type": "number", + "description": "how many pipelines have their steps asked at once; a pipeline started over it waits its turn in the state queue, placed in order on every end (design 47 H section 748).", + "minimum": 1, + "maximum": 8, + "trusted": false, + "default": 1 + }, + "pipelines-waiting-cap": { + "type": "number", + "description": "how many pipelines may wait their turn under pipelines-at-once; a start over it raises the finding queue-full and is taken on the next end (design 47 H section 748).", + "minimum": 1, + "trusted": false, + "default": 64 + }, "run-untaken-bound": { "type": "string", "description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).", @@ -145,6 +160,10 @@ "history": 1, "max-mib": 128 }, + { + "name": "queue", + "history": 1 + }, { "name": "aged", "history": 1 @@ -271,6 +290,22 @@ "mode": "0600", "content": "${config:revisions-kept}" }, + { + "id": "pipelines-at-once", + "type": "file", + "path": "${dir:state}/pipelines-at-once", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:pipelines-at-once}" + }, + { + "id": "pipelines-waiting-cap", + "type": "file", + "path": "${dir:state}/pipelines-waiting-cap", + "owner": "mesh-delivery-pipeline", + "mode": "0600", + "content": "${config:pipelines-waiting-cap}" + }, { "id": "pipeline", "type": "process", @@ -286,7 +321,9 @@ "MESH_RUNNER_TEST_STEP_BOUND_FILE": "${dir:state}/test-step-bound", "MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound", "MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept", - "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound" + "MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound", + "MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once", + "MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap" }, "user": "mesh-delivery-pipeline", "health": {