diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go index 63bdc4df..57d42256 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go @@ -19,7 +19,7 @@ import ( // §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class // act and takes §391's default callers. func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool { - h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf, + h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf, Log: func(e seat.LogEntry) error { _, err := log.Append(keys.EscapeName(e.Pipeline), e) return err @@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand } } -// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append. +// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through +// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187). type stdioPipelines struct{ s stdio.KeptState } -func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) { +func (s stdioPipelines) Get(key string) ([]byte, uint64, error) { e, err := s.s.Get(key) if err != nil || e == nil { - return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline")) + return nil, 0, err } - var p seat.Pipeline - return p, e.Revision, json.Unmarshal(e.Value, &p) + return e.Value, e.Revision, nil } -func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) { - r, err := s.s.Update(key, p, rev) +func (s stdioPipelines) Create(key string, value []byte) (uint64, error) { + r, err := s.s.Create(key, json.RawMessage(value)) + if err != nil && strings.Contains(err.Error(), "exists") { + return 0, pipeline.ErrExists + } + return r, err +} + +func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) { + r, err := s.s.Update(key, json.RawMessage(value), rev) if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) { return 0, pipeline.ErrChanged } return r, err } +func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) } + func (s stdioPipelines) Keys(filter string) ([]string, error) { all, err := s.s.Keys() var out []string diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go index 25816a91..71e7b95e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go @@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) { func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) } // newAger is the ageing over the runner's own stores. -func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { +func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged}, - Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, + Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, Outputs: func(name string) error { o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs)) if err != nil { @@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, }} } -// pipelineDeleter is the pipelines store with the delete only its owner makes (§330). -type pipelineDeleter struct{ pipelineStore } - -func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) } - // storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs // (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486). func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 4a8e3877..9488c743 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return err } - hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) + store := newPipelineStore(ctx, pipelines, logger.Printf) + hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf) hearer.Asker = busAsker{conn: conn, cred: cred} - hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf) + hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf) // The stores' fill is said before a store is full, and a full one refuses a start (§366). fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf, Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }} @@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. - starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, + starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} @@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return } // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) } - n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) + n, err := starter.Resume(store, branch, half) if err != nil { logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) return @@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). advance := func(branch string) { rv.Holders.refresh() - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) } } advancer.Store(&advance) // A registration-changed places the steps of its module's pipelines started before it, from it (A-327). register := func(reg catalog.Registration) { - if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err) } } registrar.Store(®ister) - starter.Store = pipelineStore{ctx: ctx, kv: pipelines} + starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. atOnce, waitingCap, err := queueSettings(getenv) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index d16a1cb1..6b319931 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st if err != nil { t.Fatal(err) } + // A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own. + ks = slices.DeleteFunc(ks, pipeline.IsStepsKey) slices.Sort(ks) return ks } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 5f7ce802..b5bf1b95 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" "strings" @@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed }) + // H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a + // record written before it, its steps inline, is read whole and split at its next write. + shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"}) + if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" { + t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps)) + } + inline := p + inline.Pipeline = "main.2" + inlineRaw, _ := json.Marshal(inline) + if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil { + t.Fatal(err) + } + store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) }) + old, oldRev, err := store.Get("main.2") + if err != nil || len(old.Steps) != len(p.Steps) { + t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps)) + } + if _, err := store.Update("main.2", old, oldRev); err != nil { + t.Fatal(err) + } + var split seat.Pipeline + if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) { + t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps)) + } + if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") { + t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys")) + } + // §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it. logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100) if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key { @@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult { return seat.StepResult{} } +// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to +// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog. func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool { e, err := kv.Get(ctx, key) if err != nil { return false } - *p = seat.Pipeline{} - return json.Unmarshal(e.Value(), p) == nil + var record seat.Pipeline + if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 { + panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps))) + } + got, _, err := newPipelineStore(ctx, kv, nil).Get(key) + *p = got + return err == nil } // waitFor asks cond until it holds or the test's context ends, the module's journal in the failure. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 008b8d4d..5be6db24 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,9 +1,7 @@ package main import ( - "context" "encoding/json" - "errors" "github.com/nats-io/nats.go/jetstream" @@ -13,20 +11,6 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). -type createdKV struct { - ctx context.Context - kv jetstream.KeyValue -} - -func (c createdKV) Create(key string, value []byte) error { - _, err := c.kv.Create(c.ctx, key, value) - if errors.Is(err, jetstream.ErrKeyExists) { - return pipeline.ErrExists - } - return err -} - // hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one // pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is // written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d19683fe..d6915534 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -19,64 +19,74 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27). -type pipelineStore struct { +// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines +// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187). +type kvState struct { ctx context.Context kv jetstream.KeyValue } -func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) { - e, err := s.kv.Get(s.ctx, key) - if err != nil { - return seat.Pipeline{}, 0, err - } - var p seat.Pipeline - if err := json.Unmarshal(e.Value(), &p); err != nil { - return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) - } - return p, e.Revision(), nil +// newPipelineStore is the process's one store of the state pipelines. +func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore { + return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf} } -func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { - raw, err := json.Marshal(p) - if err != nil { - return 0, err +func (s kvState) Get(key string) ([]byte, uint64, error) { + e, err := s.kv.Get(s.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil } - rev, err := s.kv.Update(s.ctx, key, raw, revision) + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (s kvState) Create(key string, value []byte) (uint64, error) { + rev, err := s.kv.Create(s.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, pipeline.ErrExists + } + return rev, err +} + +func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := s.kv.Update(s.ctx, key, value, revision) if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { return 0, pipeline.ErrChanged } return rev, err } -func (s pipelineStore) Keys(filter string) ([]string, error) { - all, err := s.kv.Keys(s.ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil +func (s kvState) Delete(key string) error { + if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err } + return nil +} + +// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects. +func (s kvState) Keys(filter string) ([]string, error) { + l, err := s.kv.ListKeysFiltered(s.ctx, filter) + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return nil, err + } + // The lister ends its own watch once the current keys are listed. var out []string - want := strings.Split(filter, ".") - for _, k := range all { - got := strings.Split(k, ".") - if len(got) != len(want) { - continue - } - ok := true - for i := range want { - ok = ok && (want[i] == "*" || want[i] == got[i]) - } - if ok { - out = append(out, k) - } + for k := range l.Keys() { + out = append(out, k) } - return out, err + return out, nil } // newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its // objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly. -func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue, +func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue, say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer { - return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module, + return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module, Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append, Keep: func(key string, k seat.KeptResult) error { raw, err := json.Marshal(k) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go new file mode 100644 index 00000000..06444930 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -0,0 +1,353 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its +// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under +// `steps..`, written by compare-and-set as the record is, so a record never grows with the catalog and no +// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names. + +// StepsKey is the key of one mover's steps in the state pipelines, `steps..`, the mover's module escaped +// as a key's name is (H-A-187). +func StepsKey(pipeline, mover string) string { + return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover)) +} + +// stepsPrefix is the start of every mover key of one pipeline. +func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") } + +// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a +// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover. +func IsStepsKey(key string) bool { + parts := strings.Split(key, ".") + if parts[0] != "steps" { + return false + } + switch len(parts) { + case 4, 7: + return true + case 5: + return parts[2] == runToken + } + return false +} + +// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by +// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a +// filter, `*` one key part. +type RawState interface { + Get(key string) ([]byte, uint64, error) + Create(key string, value []byte) (uint64, error) + Update(key string, value []byte, revision uint64) (uint64, error) + Delete(key string) error + Keys(filter string) ([]string, error) +} + +// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold. +var ErrNoPipeline = errors.New("no such pipeline") + +// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its +// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the +// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one +// pipeline take its lock, so the process never reads a record and mover keys of two writes. +type MoverStore struct { + Raw RawState + Logf func(string, ...any) + + mu sync.Mutex + locks map[string]*sync.Mutex + read map[string]readAt + inline map[string]bool +} + +// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision. +type readAt struct { + rev uint64 + movers map[string]uint64 +} + +func (s *MoverStore) lock(key string) func() { + s.mu.Lock() + if s.locks == nil { + s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{} + } + l, ok := s.locks[key] + if !ok { + l = &sync.Mutex{} + s.locks[key] = l + } + s.mu.Unlock() + l.Lock() + return l.Unlock +} + +func (s *MoverStore) logf(f string, a ...any) { + if s.Logf != nil { + s.Logf(f, a...) + } +} + +// moverKeys are the mover keys of pipeline key with their values and revisions. +func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) { + ks, err := s.Raw.Keys(stepsPrefix(key) + "*") + if err != nil { + return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err) + } + vals, revs := map[string][]byte{}, map[string]uint64{} + for _, k := range ks { + v, rev, err := s.Raw.Get(k) + if err != nil { + return nil, nil, fmt.Errorf("%s is not read: %w", k, err) + } + if v != nil { + vals[k], revs[k] = v, rev + } + } + return vals, revs, nil +} + +// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order. +func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error { + var names []string + for k := range vals { + names = append(names, k) + } + sort.Strings(names) + order := make([]string, 0, len(names)) + for _, m := range p.Movers { + if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) { + order = append(order, k) + } + } + for _, k := range names { + if !slices.Contains(order, k) { + order = append(order, k) + } + } + steps := []seat.StepResult{} + for _, k := range order { + v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k]) + if err != nil { + return fmt.Errorf("%s holds no mover's steps: %w", k, err) + } + steps = append(steps, v...) + } + p.Steps = steps + return nil +} + +// split is p's record, its own steps empty, and each mover's steps by its mover key. +func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) { + by := map[string][]seat.StepResult{} + for _, st := range p.Steps { + if st.Module == "" { + return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key) + } + k := StepsKey(key, st.Module) + by[k] = append(by[k], st) + } + movers := map[string][]byte{} + for k, steps := range by { + raw, err := json.Marshal(steps) + if err != nil { + return nil, nil, err + } + movers[k] = raw + } + p.Steps = []seat.StepResult{} + record, err := json.Marshal(p) + return record, movers, err +} + +// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one +// written before H-A-187) is read whole, said once in the journal, and split at its next write. +func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) { + defer s.lock(key)() + raw, rev, err := s.Raw.Get(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + if raw == nil { + return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + var p seat.Pipeline + if err := json.Unmarshal(raw, &p); err != nil { + return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + s.read[key] = readAt{rev: rev, movers: revs} + if len(p.Steps) > 0 { + if !s.inline[key] { + s.inline[key] = true + s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps)) + } + return p, rev, nil + } + if p.Steps == nil { + p.Steps = []seat.StepResult{} + } + return p, rev, composeSteps(&p, key, vals) +} + +// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged, +// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its +// mover keys written before the record, so its steps are never lost between the two. +func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return 0, err + } + raw, cur, err := s.Raw.Get(key) + if err != nil { + return 0, err + } + if raw == nil { + return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + if cur != revision { + return 0, ErrChanged + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return 0, err + } + // A mover key written since the read is another write between: the read named its revision. + if seen, ok := s.read[key]; ok && seen.rev == revision { + for k, r := range revs { + if seen.movers[k] != r { + return 0, ErrChanged + } + } + for k := range seen.movers { + if _, ok := revs[k]; !ok { + return 0, ErrChanged + } + } + } + var was seat.Pipeline + inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0 + var after map[string]uint64 + if inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } + rev, err := s.Raw.Update(key, record, revision) + if err != nil { + return 0, err + } + if !inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } else { + s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want)) + delete(s.inline, key) + } + s.read[key] = readAt{rev: rev, movers: after} + return rev, nil +} + +// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the +// revisions read, and answers every mover key's revision after it. +func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) { + after := map[string]uint64{} + for _, k := range slices.Sorted(maps.Keys(want)) { + v := want[k] + old, ok := vals[k] + var rev uint64 + var err error + switch { + case ok && string(old) == string(v): + after[k] = revs[k] + continue + case ok: + rev, err = s.Raw.Update(k, v, revs[k]) + default: + rev, err = s.Raw.Create(k, v) + } + if err != nil { + return nil, fmt.Errorf("the steps of %s are not written: %w", k, err) + } + after[k] = rev + } + for k := range vals { + if _, ok := want[k]; !ok { + if err := s.Raw.Delete(k); err != nil { + return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err) + } + } + } + return after, nil +} + +// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing, +// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal. +func (s *MoverStore) Create(key string, value []byte) error { + var p seat.Pipeline + if err := json.Unmarshal(value, &p); err != nil { + return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return err + } + rev, err := s.Raw.Create(key, record) + if err != nil { + return err + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return err + } + for k := range vals { + s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k) + } + after, err := s.writeMovers(key, want, vals, revs) + if err != nil { + return err + } + s.read[key] = readAt{rev: rev, movers: after} + return nil +} + +// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the +// next ageing deletes again. +func (s *MoverStore) Delete(key string) error { + unlock := s.lock(key) + vals, _, err := s.moverKeys(key) + if err == nil { + for k := range vals { + if err = s.Raw.Delete(k); err != nil { + break + } + } + } + if err == nil { + err = s.Raw.Delete(key) + } + delete(s.read, key) + delete(s.inline, key) + unlock() + return err +} + +// Keys are the keys of the state pipelines matching filter, `*` one key part. +func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go new file mode 100644 index 00000000..3f8405e2 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go @@ -0,0 +1,211 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has. +type memRaw struct { + vals map[string][]byte + revs map[string]uint64 + seq uint64 +} + +func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} } + +func (m *memRaw) Get(key string) ([]byte, uint64, error) { + v, ok := m.vals[key] + if !ok { + return nil, 0, nil + } + return append([]byte(nil), v...), m.revs[key], nil +} + +func (m *memRaw) put(key string, value []byte) uint64 { + m.seq++ + m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq + return m.seq +} + +func (m *memRaw) Create(key string, value []byte) (uint64, error) { + if _, ok := m.vals[key]; ok { + return 0, ErrExists + } + return m.put(key, value), nil +} + +func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) { + if m.revs[key] != rev { + return 0, ErrChanged + } + return m.put(key, value), nil +} + +func (m *memRaw) Delete(key string) error { + delete(m.vals, key) + delete(m.revs, key) + return nil +} + +func (m *memRaw) Keys(filter string) ([]string, error) { + var out []string + for k := range m.vals { + if ok, _ := matchKey(filter, k); ok { + out = append(out, k) + } + } + return out, nil +} + +// recordSteps is the steps the raw record of key carries itself. +func recordSteps(t *testing.T, m *memRaw, key string) int { + t.Helper() + var p seat.Pipeline + if err := json.Unmarshal(m.vals[key], &p); err != nil { + t.Fatal(err) + } + return len(p.Steps) +} + +// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's +// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline +// read whole and split at its next write, one row per case. +func TestMoverKeys(t *testing.T) { + const key = "main.7" + two := func() seat.Pipeline { + return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}}, + Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}} + } + created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + if err := s.Create(key, raw); err != nil { + t.Fatal(err) + } + return m, s, logs + } + steps := func(p seat.Pipeline) string { + var out []string + for _, st := range p.Steps { + out = append(out, st.Module+" "+st.Step+" "+st.Outcome) + } + return strings.Join(out, "; ") + } + rows := []struct { + name string + run func(t *testing.T) string + want string + }{ + {"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string { + m, s, _ := created(t) + p, _, err := s.Get(key) + return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil, + m.vals["steps.main.7.beta"] != nil, steps(p), err) + }, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running "}, + {"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + beta := m.revs["steps.main.7.beta"] + p.Steps[2].Outcome = seat.StepResultOutcomePassed + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err) + }, "beta kept true: beta build running; beta approval not asked; alpha build passed "}, + {"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"]) + p.Outcome = seat.PipelineOutcomePassed + _, err := s.Update(key, p, rev) + return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev) + }, "true, record at true"}, + {"a record written since the read refuses the update", func(t *testing.T) string { + _, s, _ := created(t) + p, rev, _ := s.Get(key) + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + _, err := s.Update(key, p, rev) + return fmt.Sprint(errors.Is(err, ErrChanged)) + }, "true"}, + {"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + m.put(key, raw) + p, rev, _ := s.Get(key) + _, _, _ = s.Get(key) + read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs)) + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q), + strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err) + }, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true "}, + {"a second create writes nothing", func(t *testing.T) string { + m, s, _ := created(t) + p := two() + p.Steps[0].Outcome = seat.StepResultOutcomeFailed + raw, _ := json.Marshal(p) + err := s.Create(key, raw) + q, _, _ := s.Get(key) + return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q)) + }, "true 3: beta build running; beta approval not asked; alpha build running"}, + {"a delete removes the mover keys and the record", func(t *testing.T) string { + m, s, _ := created(t) + err := s.Delete(key) + return fmt.Sprintf("%d %v", len(m.vals), err) + }, "0 "}, + {"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + p.Movers, p.Steps = nil, nil + for i := range 129 { + mod := fmt.Sprintf("module-%03d", i) + p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod}) + for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit, + seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition, + seat.StepResultStepReview, seat.StepResultStepApproval} { + p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program", + Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)}) + } + } + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + largest := 0 + for k, v := range m.vals { + if IsStepsKey(k) && len(v) > largest { + largest = len(v) + } + } + q, _, _ := s.Get(key) + return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10, + largest < 16<<10, len(q.Steps)) + }, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"}, + {"mover keys are told from records of every key form", func(t *testing.T) string { + var out []string + for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha", + "main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} { + out = append(out, fmt.Sprint(IsStepsKey(k))) + } + return strings.Join(out, " ") + }, "true true true false false false false"}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + if got := r.run(t); got != r.want { + t.Errorf("got %s\nwant %s", got, r.want) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index bdf25787..2b7c1392 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) { return f, nil } -// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83). +// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the +// pipeline's record with that mover's steps alone (H-A-187). func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { p, ok, err := r.pipeline(in.Pipeline) if err != nil { @@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { if err != nil { return seat.ShowAnswer{}, err } + // Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers. + if m := strings.TrimSpace(in.Mover); m != "" { + if !Moves(p, m) { + return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m) + } + p.Steps = stepsOf(p, m) + } a := seat.ShowAnswer{Pipeline: p, Fill: fills} for _, s := range p.Steps { if s.Output != "" { @@ -178,6 +186,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if key == "" { return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, ..") } + all, err := r.Pipelines.Keys() + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err) + } + return r.pipelineOf(key, all) +} + +// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that +// still carries its steps inline is read whole. +func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) { raw, err := r.Pipelines.Value(key) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err) @@ -192,7 +210,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if p.Pipeline == "" { p.Pipeline = key } - return p, true, nil + if len(p.Steps) > 0 { + return p, true, nil + } + prefix := stepsPrefix(key) + vals := map[string][]byte{} + for _, k := range all { + if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") { + continue + } + v, err := r.Pipelines.Value(k) + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err) + } + if v != nil { + vals[k] = v + } + } + return p, true, composeSteps(&p, key, vals) } func (r Reads) pipelines() ([]seat.Pipeline, error) { @@ -203,7 +238,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) { sort.Strings(keys) var out []seat.Pipeline for _, k := range keys { - p, ok, err := r.pipeline(k) + if IsStepsKey(k) { + continue + } + p, ok, err := r.pipelineOf(k, keys) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index ca728003..a7ab842c 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -3,6 +3,7 @@ package pipeline import ( "encoding/json" "errors" + "fmt" "strings" "testing" "time" @@ -49,6 +50,14 @@ func TestReads(t *testing.T) { Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}}) put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}}) + // A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps... + put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning, + Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}}) + put(t, pipes, "steps.next.8.epsilon", []seat.StepResult{{Module: "epsilon", Step: seat.StepResultStepBuild, Status: "required", + Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}) + put(t, pipes, "steps.next.8.delta", []seat.StepResult{{Module: "delta", Step: seat.StepResultStepBuild, Status: "required", + Outcome: seat.StepResultOutcomePassed}, {Module: "delta", Step: seat.StepResultStepApproval, Status: "required", + Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}}) put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"}) put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"}) r := Reads{Pipelines: pipes, Kept: kept} @@ -73,6 +82,30 @@ func TestReads(t *testing.T) { _, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"}) return "", err }, `error: no pipeline "main.9.alpha" in pipelines`}, + {"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err + }, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"}, + {"show naming a mover answers its steps alone", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err + }, "epsilon build, 2 movers"}, + {"show refuses a mover the pipeline does not move", func() (string, error) { + _, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"}) + return "", err + }, "error: the pipeline next.8 does not move alpha"}, + {"list names a pipeline, never a mover key", func() (string, error) { + a, err := r.List(seat.ListInput{Branch: "next"}) + return names(a.Pipelines), err + }, "next.8"}, {"list by module, newest first", func() (string, error) { a, err := r.List(seat.ListInput{Module: "alpha"}) return names(a.Pipelines), err @@ -92,7 +125,7 @@ func TestReads(t *testing.T) { s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04")) } return strings.Join(s, ";"), err - }, "main.7.alpha approval person 10:01"}, + }, "main.7.alpha approval person 10:01;next.8 approval person 10:02"}, {"why-carried names where a carried step came from", func() (string, error) { a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"}) return a.CarriedFrom, err