From 4e585f02b19bbed04856a942185c9dd06bb11b84 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 21:11:59 +0100 Subject: [PATCH] Design 47 H1: the runner keeps each mover's steps under its own key, so a record never grows with the catalog (H-A-187) The runner's state pipelines held a pipeline's every step in its record, and a record of 112 or 129 movers exceeded the bus's 1 MiB message limit (ADR 0486), so main.26 and main.30 placed no step. One MoverStore per process now reads and writes the state: the record keeps the movers, tiers, call, source and outcome with its own steps empty, each mover's steps are written under steps.. by compare-and-set as the record is, and nothing is written when the record or a mover key changed since the read. Every read and judgement of steps goes through it, so the placing, the step ends, the end judgement, the hand acts and the ageing see one composed pipeline; show, list, stands and why-carried compose from the mover keys, and show naming a mover answers its steps alone. A record still carrying its steps inline is read whole, said once in the journal, and split at its next write, its mover keys written before the record. Builds: design 47 H H-A-187 --- .../mesh-delivery-pipeline-tools/unlogged.go | 26 +- .../cmd/mesh-delivery-pipeline/aged.go | 9 +- .../cmd/mesh-delivery-pipeline/main.go | 17 +- .../queue_integration_test.go | 2 + .../run_integration_test.go | 40 +- .../cmd/mesh-delivery-pipeline/start.go | 16 - .../cmd/mesh-delivery-pipeline/steps.go | 84 +++-- .../internal/pipeline/moverkeys.go | 353 ++++++++++++++++++ .../internal/pipeline/moverkeys_test.go | 211 +++++++++++ .../internal/pipeline/read.go | 44 ++- .../internal/pipeline/read_test.go | 35 +- 11 files changed, 755 insertions(+), 82 deletions(-) create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go create mode 100644 modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go index 63bdc4df..57d42256 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go @@ -19,7 +19,7 @@ import ( // §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class // act and takes §391's default callers. func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool { - h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf, + h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf, Log: func(e seat.LogEntry) error { _, err := log.Append(keys.EscapeName(e.Pipeline), e) return err @@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand } } -// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append. +// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through +// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187). type stdioPipelines struct{ s stdio.KeptState } -func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) { +func (s stdioPipelines) Get(key string) ([]byte, uint64, error) { e, err := s.s.Get(key) if err != nil || e == nil { - return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline")) + return nil, 0, err } - var p seat.Pipeline - return p, e.Revision, json.Unmarshal(e.Value, &p) + return e.Value, e.Revision, nil } -func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) { - r, err := s.s.Update(key, p, rev) +func (s stdioPipelines) Create(key string, value []byte) (uint64, error) { + r, err := s.s.Create(key, json.RawMessage(value)) + if err != nil && strings.Contains(err.Error(), "exists") { + return 0, pipeline.ErrExists + } + return r, err +} + +func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) { + r, err := s.s.Update(key, json.RawMessage(value), rev) if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) { return 0, pipeline.ErrChanged } return r, err } +func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) } + func (s stdioPipelines) Keys(filter string) ([]string, error) { all, err := s.s.Keys() var out []string diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go index 25816a91..71e7b95e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go @@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) { func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) } // newAger is the ageing over the runner's own stores. -func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { +func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged}, - Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, + Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, Outputs: func(name string) error { o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs)) if err != nil { @@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, }} } -// pipelineDeleter is the pipelines store with the delete only its owner makes (§330). -type pipelineDeleter struct{ pipelineStore } - -func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) } - // storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs // (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486). func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 4a8e3877..9488c743 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return err } - hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) + store := newPipelineStore(ctx, pipelines, logger.Printf) + hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf) hearer.Asker = busAsker{conn: conn, cred: cred} - hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf) + hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf) // The stores' fill is said before a store is full, and a full one refuses a start (§366). fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf, Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }} @@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. - starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, + starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} @@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return } // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) } - n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) + n, err := starter.Resume(store, branch, half) if err != nil { logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) return @@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). advance := func(branch string) { rv.Holders.refresh() - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) } } advancer.Store(&advance) // A registration-changed places the steps of its module's pipelines started before it, from it (A-327). register := func(reg catalog.Registration) { - if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err) } } registrar.Store(®ister) - starter.Store = pipelineStore{ctx: ctx, kv: pipelines} + starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. atOnce, waitingCap, err := queueSettings(getenv) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index d16a1cb1..6b319931 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st if err != nil { t.Fatal(err) } + // A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own. + ks = slices.DeleteFunc(ks, pipeline.IsStepsKey) slices.Sort(ks) return ks } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 5f7ce802..b5bf1b95 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" "strings" @@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed }) + // H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a + // record written before it, its steps inline, is read whole and split at its next write. + shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"}) + if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" { + t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps)) + } + inline := p + inline.Pipeline = "main.2" + inlineRaw, _ := json.Marshal(inline) + if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil { + t.Fatal(err) + } + store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) }) + old, oldRev, err := store.Get("main.2") + if err != nil || len(old.Steps) != len(p.Steps) { + t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps)) + } + if _, err := store.Update("main.2", old, oldRev); err != nil { + t.Fatal(err) + } + var split seat.Pipeline + if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) { + t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps)) + } + if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") { + t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys")) + } + // §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it. logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100) if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key { @@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult { return seat.StepResult{} } +// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to +// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog. func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool { e, err := kv.Get(ctx, key) if err != nil { return false } - *p = seat.Pipeline{} - return json.Unmarshal(e.Value(), p) == nil + var record seat.Pipeline + if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 { + panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps))) + } + got, _, err := newPipelineStore(ctx, kv, nil).Get(key) + *p = got + return err == nil } // waitFor asks cond until it holds or the test's context ends, the module's journal in the failure. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 008b8d4d..5be6db24 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,9 +1,7 @@ package main import ( - "context" "encoding/json" - "errors" "github.com/nats-io/nats.go/jetstream" @@ -13,20 +11,6 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). -type createdKV struct { - ctx context.Context - kv jetstream.KeyValue -} - -func (c createdKV) Create(key string, value []byte) error { - _, err := c.kv.Create(c.ctx, key, value) - if errors.Is(err, jetstream.ErrKeyExists) { - return pipeline.ErrExists - } - return err -} - // hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one // pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is // written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d19683fe..d6915534 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -19,64 +19,74 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27). -type pipelineStore struct { +// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines +// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187). +type kvState struct { ctx context.Context kv jetstream.KeyValue } -func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) { - e, err := s.kv.Get(s.ctx, key) - if err != nil { - return seat.Pipeline{}, 0, err - } - var p seat.Pipeline - if err := json.Unmarshal(e.Value(), &p); err != nil { - return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) - } - return p, e.Revision(), nil +// newPipelineStore is the process's one store of the state pipelines. +func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore { + return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf} } -func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { - raw, err := json.Marshal(p) - if err != nil { - return 0, err +func (s kvState) Get(key string) ([]byte, uint64, error) { + e, err := s.kv.Get(s.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil } - rev, err := s.kv.Update(s.ctx, key, raw, revision) + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (s kvState) Create(key string, value []byte) (uint64, error) { + rev, err := s.kv.Create(s.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, pipeline.ErrExists + } + return rev, err +} + +func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := s.kv.Update(s.ctx, key, value, revision) if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { return 0, pipeline.ErrChanged } return rev, err } -func (s pipelineStore) Keys(filter string) ([]string, error) { - all, err := s.kv.Keys(s.ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil +func (s kvState) Delete(key string) error { + if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err } + return nil +} + +// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects. +func (s kvState) Keys(filter string) ([]string, error) { + l, err := s.kv.ListKeysFiltered(s.ctx, filter) + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return nil, err + } + // The lister ends its own watch once the current keys are listed. var out []string - want := strings.Split(filter, ".") - for _, k := range all { - got := strings.Split(k, ".") - if len(got) != len(want) { - continue - } - ok := true - for i := range want { - ok = ok && (want[i] == "*" || want[i] == got[i]) - } - if ok { - out = append(out, k) - } + for k := range l.Keys() { + out = append(out, k) } - return out, err + return out, nil } // newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its // objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly. -func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue, +func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue, say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer { - return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module, + return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module, Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append, Keep: func(key string, k seat.KeptResult) error { raw, err := json.Marshal(k) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go new file mode 100644 index 00000000..06444930 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -0,0 +1,353 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its +// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under +// `steps..`, written by compare-and-set as the record is, so a record never grows with the catalog and no +// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names. + +// StepsKey is the key of one mover's steps in the state pipelines, `steps..`, the mover's module escaped +// as a key's name is (H-A-187). +func StepsKey(pipeline, mover string) string { + return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover)) +} + +// stepsPrefix is the start of every mover key of one pipeline. +func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") } + +// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a +// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover. +func IsStepsKey(key string) bool { + parts := strings.Split(key, ".") + if parts[0] != "steps" { + return false + } + switch len(parts) { + case 4, 7: + return true + case 5: + return parts[2] == runToken + } + return false +} + +// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by +// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a +// filter, `*` one key part. +type RawState interface { + Get(key string) ([]byte, uint64, error) + Create(key string, value []byte) (uint64, error) + Update(key string, value []byte, revision uint64) (uint64, error) + Delete(key string) error + Keys(filter string) ([]string, error) +} + +// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold. +var ErrNoPipeline = errors.New("no such pipeline") + +// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its +// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the +// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one +// pipeline take its lock, so the process never reads a record and mover keys of two writes. +type MoverStore struct { + Raw RawState + Logf func(string, ...any) + + mu sync.Mutex + locks map[string]*sync.Mutex + read map[string]readAt + inline map[string]bool +} + +// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision. +type readAt struct { + rev uint64 + movers map[string]uint64 +} + +func (s *MoverStore) lock(key string) func() { + s.mu.Lock() + if s.locks == nil { + s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{} + } + l, ok := s.locks[key] + if !ok { + l = &sync.Mutex{} + s.locks[key] = l + } + s.mu.Unlock() + l.Lock() + return l.Unlock +} + +func (s *MoverStore) logf(f string, a ...any) { + if s.Logf != nil { + s.Logf(f, a...) + } +} + +// moverKeys are the mover keys of pipeline key with their values and revisions. +func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) { + ks, err := s.Raw.Keys(stepsPrefix(key) + "*") + if err != nil { + return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err) + } + vals, revs := map[string][]byte{}, map[string]uint64{} + for _, k := range ks { + v, rev, err := s.Raw.Get(k) + if err != nil { + return nil, nil, fmt.Errorf("%s is not read: %w", k, err) + } + if v != nil { + vals[k], revs[k] = v, rev + } + } + return vals, revs, nil +} + +// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order. +func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error { + var names []string + for k := range vals { + names = append(names, k) + } + sort.Strings(names) + order := make([]string, 0, len(names)) + for _, m := range p.Movers { + if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) { + order = append(order, k) + } + } + for _, k := range names { + if !slices.Contains(order, k) { + order = append(order, k) + } + } + steps := []seat.StepResult{} + for _, k := range order { + v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k]) + if err != nil { + return fmt.Errorf("%s holds no mover's steps: %w", k, err) + } + steps = append(steps, v...) + } + p.Steps = steps + return nil +} + +// split is p's record, its own steps empty, and each mover's steps by its mover key. +func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) { + by := map[string][]seat.StepResult{} + for _, st := range p.Steps { + if st.Module == "" { + return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key) + } + k := StepsKey(key, st.Module) + by[k] = append(by[k], st) + } + movers := map[string][]byte{} + for k, steps := range by { + raw, err := json.Marshal(steps) + if err != nil { + return nil, nil, err + } + movers[k] = raw + } + p.Steps = []seat.StepResult{} + record, err := json.Marshal(p) + return record, movers, err +} + +// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one +// written before H-A-187) is read whole, said once in the journal, and split at its next write. +func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) { + defer s.lock(key)() + raw, rev, err := s.Raw.Get(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + if raw == nil { + return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + var p seat.Pipeline + if err := json.Unmarshal(raw, &p); err != nil { + return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + s.read[key] = readAt{rev: rev, movers: revs} + if len(p.Steps) > 0 { + if !s.inline[key] { + s.inline[key] = true + s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps)) + } + return p, rev, nil + } + if p.Steps == nil { + p.Steps = []seat.StepResult{} + } + return p, rev, composeSteps(&p, key, vals) +} + +// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged, +// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its +// mover keys written before the record, so its steps are never lost between the two. +func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return 0, err + } + raw, cur, err := s.Raw.Get(key) + if err != nil { + return 0, err + } + if raw == nil { + return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + if cur != revision { + return 0, ErrChanged + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return 0, err + } + // A mover key written since the read is another write between: the read named its revision. + if seen, ok := s.read[key]; ok && seen.rev == revision { + for k, r := range revs { + if seen.movers[k] != r { + return 0, ErrChanged + } + } + for k := range seen.movers { + if _, ok := revs[k]; !ok { + return 0, ErrChanged + } + } + } + var was seat.Pipeline + inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0 + var after map[string]uint64 + if inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } + rev, err := s.Raw.Update(key, record, revision) + if err != nil { + return 0, err + } + if !inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } else { + s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want)) + delete(s.inline, key) + } + s.read[key] = readAt{rev: rev, movers: after} + return rev, nil +} + +// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the +// revisions read, and answers every mover key's revision after it. +func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) { + after := map[string]uint64{} + for _, k := range slices.Sorted(maps.Keys(want)) { + v := want[k] + old, ok := vals[k] + var rev uint64 + var err error + switch { + case ok && string(old) == string(v): + after[k] = revs[k] + continue + case ok: + rev, err = s.Raw.Update(k, v, revs[k]) + default: + rev, err = s.Raw.Create(k, v) + } + if err != nil { + return nil, fmt.Errorf("the steps of %s are not written: %w", k, err) + } + after[k] = rev + } + for k := range vals { + if _, ok := want[k]; !ok { + if err := s.Raw.Delete(k); err != nil { + return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err) + } + } + } + return after, nil +} + +// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing, +// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal. +func (s *MoverStore) Create(key string, value []byte) error { + var p seat.Pipeline + if err := json.Unmarshal(value, &p); err != nil { + return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return err + } + rev, err := s.Raw.Create(key, record) + if err != nil { + return err + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return err + } + for k := range vals { + s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k) + } + after, err := s.writeMovers(key, want, vals, revs) + if err != nil { + return err + } + s.read[key] = readAt{rev: rev, movers: after} + return nil +} + +// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the +// next ageing deletes again. +func (s *MoverStore) Delete(key string) error { + unlock := s.lock(key) + vals, _, err := s.moverKeys(key) + if err == nil { + for k := range vals { + if err = s.Raw.Delete(k); err != nil { + break + } + } + } + if err == nil { + err = s.Raw.Delete(key) + } + delete(s.read, key) + delete(s.inline, key) + unlock() + return err +} + +// Keys are the keys of the state pipelines matching filter, `*` one key part. +func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go new file mode 100644 index 00000000..3f8405e2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go @@ -0,0 +1,211 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has. +type memRaw struct { + vals map[string][]byte + revs map[string]uint64 + seq uint64 +} + +func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} } + +func (m *memRaw) Get(key string) ([]byte, uint64, error) { + v, ok := m.vals[key] + if !ok { + return nil, 0, nil + } + return append([]byte(nil), v...), m.revs[key], nil +} + +func (m *memRaw) put(key string, value []byte) uint64 { + m.seq++ + m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq + return m.seq +} + +func (m *memRaw) Create(key string, value []byte) (uint64, error) { + if _, ok := m.vals[key]; ok { + return 0, ErrExists + } + return m.put(key, value), nil +} + +func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) { + if m.revs[key] != rev { + return 0, ErrChanged + } + return m.put(key, value), nil +} + +func (m *memRaw) Delete(key string) error { + delete(m.vals, key) + delete(m.revs, key) + return nil +} + +func (m *memRaw) Keys(filter string) ([]string, error) { + var out []string + for k := range m.vals { + if ok, _ := matchKey(filter, k); ok { + out = append(out, k) + } + } + return out, nil +} + +// recordSteps is the steps the raw record of key carries itself. +func recordSteps(t *testing.T, m *memRaw, key string) int { + t.Helper() + var p seat.Pipeline + if err := json.Unmarshal(m.vals[key], &p); err != nil { + t.Fatal(err) + } + return len(p.Steps) +} + +// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's +// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline +// read whole and split at its next write, one row per case. +func TestMoverKeys(t *testing.T) { + const key = "main.7" + two := func() seat.Pipeline { + return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}}, + Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}} + } + created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + if err := s.Create(key, raw); err != nil { + t.Fatal(err) + } + return m, s, logs + } + steps := func(p seat.Pipeline) string { + var out []string + for _, st := range p.Steps { + out = append(out, st.Module+" "+st.Step+" "+st.Outcome) + } + return strings.Join(out, "; ") + } + rows := []struct { + name string + run func(t *testing.T) string + want string + }{ + {"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string { + m, s, _ := created(t) + p, _, err := s.Get(key) + return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil, + m.vals["steps.main.7.beta"] != nil, steps(p), err) + }, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running "}, + {"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + beta := m.revs["steps.main.7.beta"] + p.Steps[2].Outcome = seat.StepResultOutcomePassed + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err) + }, "beta kept true: beta build running; beta approval not asked; alpha build passed "}, + {"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"]) + p.Outcome = seat.PipelineOutcomePassed + _, err := s.Update(key, p, rev) + return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev) + }, "true, record at true"}, + {"a record written since the read refuses the update", func(t *testing.T) string { + _, s, _ := created(t) + p, rev, _ := s.Get(key) + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + _, err := s.Update(key, p, rev) + return fmt.Sprint(errors.Is(err, ErrChanged)) + }, "true"}, + {"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + m.put(key, raw) + p, rev, _ := s.Get(key) + _, _, _ = s.Get(key) + read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs)) + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q), + strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err) + }, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true "}, + {"a second create writes nothing", func(t *testing.T) string { + m, s, _ := created(t) + p := two() + p.Steps[0].Outcome = seat.StepResultOutcomeFailed + raw, _ := json.Marshal(p) + err := s.Create(key, raw) + q, _, _ := s.Get(key) + return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q)) + }, "true 3: beta build running; beta approval not asked; alpha build running"}, + {"a delete removes the mover keys and the record", func(t *testing.T) string { + m, s, _ := created(t) + err := s.Delete(key) + return fmt.Sprintf("%d %v", len(m.vals), err) + }, "0 "}, + {"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + p.Movers, p.Steps = nil, nil + for i := range 129 { + mod := fmt.Sprintf("module-%03d", i) + p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod}) + for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit, + seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition, + seat.StepResultStepReview, seat.StepResultStepApproval} { + p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program", + Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)}) + } + } + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + largest := 0 + for k, v := range m.vals { + if IsStepsKey(k) && len(v) > largest { + largest = len(v) + } + } + q, _, _ := s.Get(key) + return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10, + largest < 16<<10, len(q.Steps)) + }, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"}, + {"mover keys are told from records of every key form", func(t *testing.T) string { + var out []string + for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha", + "main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} { + out = append(out, fmt.Sprint(IsStepsKey(k))) + } + return strings.Join(out, " ") + }, "true true true false false false false"}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + if got := r.run(t); got != r.want { + t.Errorf("got %s\nwant %s", got, r.want) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index bdf25787..2b7c1392 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) { return f, nil } -// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83). +// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the +// pipeline's record with that mover's steps alone (H-A-187). func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { p, ok, err := r.pipeline(in.Pipeline) if err != nil { @@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { if err != nil { return seat.ShowAnswer{}, err } + // Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers. + if m := strings.TrimSpace(in.Mover); m != "" { + if !Moves(p, m) { + return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m) + } + p.Steps = stepsOf(p, m) + } a := seat.ShowAnswer{Pipeline: p, Fill: fills} for _, s := range p.Steps { if s.Output != "" { @@ -178,6 +186,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if key == "" { return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, ..") } + all, err := r.Pipelines.Keys() + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err) + } + return r.pipelineOf(key, all) +} + +// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that +// still carries its steps inline is read whole. +func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) { raw, err := r.Pipelines.Value(key) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err) @@ -192,7 +210,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if p.Pipeline == "" { p.Pipeline = key } - return p, true, nil + if len(p.Steps) > 0 { + return p, true, nil + } + prefix := stepsPrefix(key) + vals := map[string][]byte{} + for _, k := range all { + if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") { + continue + } + v, err := r.Pipelines.Value(k) + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err) + } + if v != nil { + vals[k] = v + } + } + return p, true, composeSteps(&p, key, vals) } func (r Reads) pipelines() ([]seat.Pipeline, error) { @@ -203,7 +238,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) { sort.Strings(keys) var out []seat.Pipeline for _, k := range keys { - p, ok, err := r.pipeline(k) + if IsStepsKey(k) { + continue + } + p, ok, err := r.pipelineOf(k, keys) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index ca728003..a7ab842c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -3,6 +3,7 @@ package pipeline import ( "encoding/json" "errors" + "fmt" "strings" "testing" "time" @@ -49,6 +50,14 @@ func TestReads(t *testing.T) { Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}}) put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}}) + // A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps... + put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning, + Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}}) + put(t, pipes, "steps.next.8.epsilon", []seat.StepResult{{Module: "epsilon", Step: seat.StepResultStepBuild, Status: "required", + Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}) + put(t, pipes, "steps.next.8.delta", []seat.StepResult{{Module: "delta", Step: seat.StepResultStepBuild, Status: "required", + Outcome: seat.StepResultOutcomePassed}, {Module: "delta", Step: seat.StepResultStepApproval, Status: "required", + Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}}) put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"}) put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"}) r := Reads{Pipelines: pipes, Kept: kept} @@ -73,6 +82,30 @@ func TestReads(t *testing.T) { _, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"}) return "", err }, `error: no pipeline "main.9.alpha" in pipelines`}, + {"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err + }, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"}, + {"show naming a mover answers its steps alone", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err + }, "epsilon build, 2 movers"}, + {"show refuses a mover the pipeline does not move", func() (string, error) { + _, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"}) + return "", err + }, "error: the pipeline next.8 does not move alpha"}, + {"list names a pipeline, never a mover key", func() (string, error) { + a, err := r.List(seat.ListInput{Branch: "next"}) + return names(a.Pipelines), err + }, "next.8"}, {"list by module, newest first", func() (string, error) { a, err := r.List(seat.ListInput{Module: "alpha"}) return names(a.Pipelines), err @@ -92,7 +125,7 @@ func TestReads(t *testing.T) { s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04")) } return strings.Join(s, ";"), err - }, "main.7.alpha approval person 10:01"}, + }, "main.7.alpha approval person 10:01;next.8 approval person 10:02"}, {"why-carried names where a carried step came from", func() (string, error) { a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"}) return a.CarriedFrom, err