diff --git a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go index ce60aa65..058625fb 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner/contract.go @@ -276,6 +276,48 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu return v, nil } +// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps..". +func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string { + return "steps." + pipeline + "." + mover +} + +// ParsePipelinesByStepsPipelineMoverKey reads the parts of a key of the form "steps..", in the order of the form; the last result is false for a key of another form. +func ParsePipelinesByStepsPipelineMoverKey(key string) (string, string, bool) { + rest := key + if !strings.HasPrefix(rest, "steps.") { + return "", "", false + } + rest = rest[6:] + v1, after1, found1 := strings.Cut(rest, ".") + if !found1 || v1 == "" { + return "", "", false + } + rest = after1 + if rest == "" { + return "", "", false + } + v3 := rest + rest = "" + if rest != "" { + return "", "", false + } + return v1, v3, true +} + +// DecodePipelinesByStepsPipelineMoverValue reads one value of the state pipelines under the form "steps..", refusing a field PipelinesByStepsPipelineMoverValue does not know and any text after the value. +func DecodePipelinesByStepsPipelineMoverValue(raw []byte) (PipelinesByStepsPipelineMoverValue, error) { + var v PipelinesByStepsPipelineMoverValue + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return v, err + } + if dec.More() { + return v, errors.New("text after the value") + } + return v, nil +} + // QueueByQueueNextKey is the key of one value of the state queue under the form "queue.next". func QueueByQueueNextKey() string { return "queue.next" } @@ -523,8 +565,10 @@ type LogEntry struct { Artifact string `json:"artifact,omitempty"` At time.Time `json:"at"` // the keys of the kept results the step wrote or carried, ., which the runner ages by their last-read (design 47 G sections 330, 470) - Kept []string `json:"kept,omitempty"` - Outcome string `json:"outcome,omitempty"` + Kept []string `json:"kept,omitempty"` + // the module of the mover whose step the entry is of (H-A-190) + Module string `json:"module,omitempty"` + Outcome string `json:"outcome,omitempty"` // the output's name in step-outputs Output string `json:"output,omitempty"` Pipeline string `json:"pipeline"` @@ -621,6 +665,8 @@ type Pipeline struct { // how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) EndCount int64 `json:"end-count,omitempty"` Ended *time.Time `json:"ended,omitempty"` + // how many movers the pipeline has, set by list alone (H-A-191) + MoverCount int64 `json:"mover-count,omitempty"` // the modules the pipeline moves, its take-in's entries (design 47 H §746) Movers []Mover `json:"movers"` Outcome string `json:"outcome"` @@ -637,11 +683,13 @@ type Pipeline struct { // the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) Source int64 `json:"source"` Started *time.Time `json:"started,omitempty"` - // every mover's steps, each naming its mover, approval last + // how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) + StepCount int64 `json:"step-count,omitempty"` + // every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) Steps []StepResult `json:"steps"` // the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) Superseded []string `json:"superseded,omitempty"` - // the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) + // the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) Unlogged []LogEntry `json:"unlogged,omitempty"` // the layout version V int64 `json:"v,omitempty"` @@ -1026,6 +1074,8 @@ type StartAnswer struct { // ShowInput is what show takes. type ShowInput struct { + // a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) + Mover string `json:"mover,omitempty"` // the pipeline's key Pipeline string `json:"pipeline"` } @@ -1314,6 +1364,14 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline // PipelinesByBranchRunRunValue is one value of the state pipelines under the key .run.. type PipelinesByBranchRunRunValue = Pipeline +// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps... +type PipelinesByStepsPipelineMoverValue struct { + // the mover's steps + Steps []StepResult `json:"steps"` + // the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) + Unlogged []LogEntry `json:"unlogged,omitempty"` +} + // QueueByQueueNextValue is one value of the state queue under the key queue.next. type QueueByQueueNextValue = int64 diff --git a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json index 7a5c3607..5bf14a52 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-pipeline-runner/seat.json @@ -86,13 +86,17 @@ }, { "name": "show", - "description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.", + "description": "One pipeline whole: its record with every mover's steps, or one mover's steps alone when it names the mover, its step log and its outputs, with the stores' fill. Reads only.", "input": { "type": "object", "properties": { "pipeline": { "type": "string", "description": "the pipeline's key" + }, + "mover": { + "type": "string", + "description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)" } }, "required": [ @@ -133,7 +137,7 @@ }, { "name": "list", - "description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.", + "description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.", "input": { "type": "object", "properties": { @@ -771,7 +775,7 @@ "name": "pipelines", "template": "state", "history": 1, - "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747)", + "description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)", "key-forms": [ { "key": ".", @@ -793,6 +797,32 @@ "$ref": "#/defs/Pipeline" }, "description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)" + }, + { + "key": "steps..", + "value": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "$ref": "#/defs/StepResult" + }, + "description": "the mover's steps" + }, + "unlogged": { + "type": "array", + "items": { + "$ref": "#/defs/LogEntry" + }, + "description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)" + } + }, + "required": [ + "steps" + ] + }, + "description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)" } ] }, @@ -1238,7 +1268,7 @@ "items": { "$ref": "#/defs/StepResult" }, - "description": "every mover's steps, each naming its mover, approval last" + "description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)" }, "started": { "type": "string", @@ -1253,7 +1283,15 @@ "items": { "$ref": "#/defs/LogEntry" }, - "description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)" + "description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)" + }, + "mover-count": { + "type": "integer", + "description": "how many movers the pipeline has, set by list alone (H-A-191)" + }, + "step-count": { + "type": "integer", + "description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)" }, "published": { "$ref": "#/defs/Published", @@ -1613,6 +1651,10 @@ "pipeline": { "type": "string" }, + "module": { + "type": "string", + "description": "the module of the mover whose step the entry is of (H-A-190)" + }, "step": { "type": "string", "enum": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts index 4fb293a0..748152b9 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.d.ts @@ -112,6 +112,12 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string): /** The parts of a key of the form ".run.", or undefined for a key of another form. */ export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined; +/** The key of one value of the state pipelines under the form "steps..". */ +export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string; + +/** The parts of a key of the form "steps..", or undefined for a key of another form. */ +export declare function parsePipelinesByStepsPipelineMoverKey(_key: string): { pipeline: string; mover: string } | undefined; + /** The key of one value of the state queue under the form "queue.next". */ export declare function queueByQueueNextKey(): string; @@ -186,6 +192,8 @@ export interface LogEntry { "at": string; /** the keys of the kept results the step wrote or carried, ., which the runner ages by their last-read (design 47 G sections 330, 470) */ "kept"?: string[]; + /** the module of the mover whose step the entry is of (H-A-190) */ + "module"?: string; "outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled"; /** the output's name in step-outputs */ "output"?: string; @@ -248,6 +256,8 @@ export interface Pipeline { /** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */ "end-count"?: number; "ended"?: string; + /** how many movers the pipeline has, set by list alone (H-A-191) */ + "mover-count"?: number; /** the modules the pipeline moves, its take-in's entries (design 47 H §746) */ "movers": Mover[]; "outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled"; @@ -264,11 +274,13 @@ export interface Pipeline { /** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */ "source": number; "started"?: string; - /** every mover's steps, each naming its mover, approval last */ + /** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */ + "step-count"?: number; + /** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */ "steps": StepResult[]; /** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */ "superseded"?: string[]; - /** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */ + /** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */ "unlogged"?: LogEntry[]; /** the layout version */ "v"?: number; @@ -556,6 +568,8 @@ export interface StartAnswer { /** is what show takes. */ export interface ShowInput { + /** a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) */ + "mover"?: string; /** the pipeline's key */ "pipeline": string; } @@ -795,6 +809,14 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline; /** is one value of the state pipelines under the key .run.. */ export type PipelinesByBranchRunRunValue = Pipeline; +/** is one value of the state pipelines under the key steps... */ +export interface PipelinesByStepsPipelineMoverValue { + /** the mover's steps */ + "steps": StepResult[]; + /** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */ + "unlogged"?: LogEntry[]; +} + /** is one value of the state queue under the key queue.next. */ export type QueueByQueueNextValue = number; diff --git a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js index 32282d84..213a3df3 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshpipelinerunner/index.js @@ -173,6 +173,27 @@ export function parsePipelinesByBranchRunRunKey(_key) { return { branch, run }; } +/** The key of one value of the state pipelines under the form "steps..". */ +export function pipelinesByStepsPipelineMoverKey(pipeline, mover) { + return "steps." + pipeline + "." + mover; +} + +/** The parts of a key of the form "steps..", or undefined for a key of another form. */ +export function parsePipelinesByStepsPipelineMoverKey(_key) { + let _rest = _key; + if (!_rest.startsWith("steps.")) return undefined; + _rest = _rest.slice(6); + const _at1 = _rest.indexOf("."); + if (_at1 <= 0) return undefined; + const pipeline = _rest.slice(0, _at1); + _rest = _rest.slice(_at1 + 1); + if (_rest === "") return undefined; + const mover = _rest; + _rest = ""; + if (_rest !== "") return undefined; + return { pipeline, mover }; +} + /** The key of one value of the state queue under the form "queue.next". */ export function queueByQueueNextKey() { return "queue.next"; diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go index 63bdc4df..57d42256 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/unlogged.go @@ -19,7 +19,7 @@ import ( // §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class // act and takes §391's default callers. func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool { - h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf, + h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf, Log: func(e seat.LogEntry) error { _, err := log.Append(keys.EscapeName(e.Pipeline), e) return err @@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand } } -// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append. +// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through +// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187). type stdioPipelines struct{ s stdio.KeptState } -func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) { +func (s stdioPipelines) Get(key string) ([]byte, uint64, error) { e, err := s.s.Get(key) if err != nil || e == nil { - return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline")) + return nil, 0, err } - var p seat.Pipeline - return p, e.Revision, json.Unmarshal(e.Value, &p) + return e.Value, e.Revision, nil } -func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) { - r, err := s.s.Update(key, p, rev) +func (s stdioPipelines) Create(key string, value []byte) (uint64, error) { + r, err := s.s.Create(key, json.RawMessage(value)) + if err != nil && strings.Contains(err.Error(), "exists") { + return 0, pipeline.ErrExists + } + return r, err +} + +func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) { + r, err := s.s.Update(key, json.RawMessage(value), rev) if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) { return 0, pipeline.ErrChanged } return r, err } +func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) } + func (s stdioPipelines) Keys(filter string) ([]string, error) { all, err := s.s.Keys() var out []string diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go index 25816a91..71e7b95e 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/aged.go @@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) { func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) } // newAger is the ageing over the runner's own stores. -func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { +func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager { return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged}, - Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, + Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf, Outputs: func(name string) error { o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs)) if err != nil { @@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, }} } -// pipelineDeleter is the pipelines store with the delete only its owner makes (§330). -type pipelineDeleter struct{ pipelineStore } - -func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) } - // storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs // (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486). func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) { diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go index 4a8e3877..9488c743 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go @@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return err } - hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf) + store := newPipelineStore(ctx, pipelines, logger.Printf) + hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf) hearer.Asker = busAsker{conn: conn, cred: cred} - hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf) + hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf) // The stores' fill is said before a store is full, and a full one refuses a start (§366). fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf, Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }} @@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf} // Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked // for its step. - starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say, + starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say, Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module, Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains), Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing} @@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return } // A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327). - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err) } - n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half) + n, err := starter.Resume(store, branch, half) if err != nil { logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err) return @@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200). advance := func(branch string) { rv.Holders.refresh() - if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err) } } advancer.Store(&advance) // A registration-changed places the steps of its module's pipelines started before it, from it (A-327). register := func(reg catalog.Registration) { - if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 { + if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 { logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err) } } registrar.Store(®ister) - starter.Store = pipelineStore{ctx: ctx, kv: pipelines} + starter.Store = store // The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in // the state queue, placed at the start from it and on every end. atOnce, waitingCap, err := queueSettings(getenv) diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go index d16a1cb1..6b319931 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/queue_integration_test.go @@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st if err != nil { t.Fatal(err) } + // A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own. + ks = slices.DeleteFunc(ks, pipeline.IsStepsKey) slices.Sort(ks) return ks } diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go index 5f7ce802..b5bf1b95 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/run_integration_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "path/filepath" "strings" @@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed }) + // H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a + // record written before it, its steps inline, is read whole and split at its next write. + shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"}) + if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" { + t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps)) + } + inline := p + inline.Pipeline = "main.2" + inlineRaw, _ := json.Marshal(inline) + if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil { + t.Fatal(err) + } + store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) }) + old, oldRev, err := store.Get("main.2") + if err != nil || len(old.Steps) != len(p.Steps) { + t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps)) + } + if _, err := store.Update("main.2", old, oldRev); err != nil { + t.Fatal(err) + } + var split seat.Pipeline + if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) { + t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps)) + } + if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") { + t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys")) + } + // §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it. logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100) if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key { @@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult { return seat.StepResult{} } +// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to +// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog. func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool { e, err := kv.Get(ctx, key) if err != nil { return false } - *p = seat.Pipeline{} - return json.Unmarshal(e.Value(), p) == nil + var record seat.Pipeline + if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 { + panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps))) + } + got, _, err := newPipelineStore(ctx, kv, nil).Get(key) + *p = got + return err == nil } // waitFor asks cond until it holds or the test's context ends, the module's journal in the failure. diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go index 008b8d4d..5be6db24 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/start.go @@ -1,9 +1,7 @@ package main import ( - "context" "encoding/json" - "errors" "github.com/nats-io/nats.go/jetstream" @@ -13,20 +11,6 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// createdKV is the state pipelines written by compare-and-set on a key's absence (§54). -type createdKV struct { - ctx context.Context - kv jetstream.KeyValue -} - -func (c createdKV) Create(key string, value []byte) error { - _, err := c.kv.Create(c.ctx, key, value) - if errors.Is(err, jetstream.ErrKeyExists) { - return pipeline.ErrExists - } - return err -} - // hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one // pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is // written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard diff --git a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go index d19683fe..d6915534 100644 --- a/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/steps.go @@ -19,64 +19,74 @@ import ( "mesh-delivery-pipeline/internal/pipeline" ) -// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27). -type pipelineStore struct { +// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines +// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187). +type kvState struct { ctx context.Context kv jetstream.KeyValue } -func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) { - e, err := s.kv.Get(s.ctx, key) - if err != nil { - return seat.Pipeline{}, 0, err - } - var p seat.Pipeline - if err := json.Unmarshal(e.Value(), &p); err != nil { - return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) - } - return p, e.Revision(), nil +// newPipelineStore is the process's one store of the state pipelines. +func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore { + return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf} } -func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { - raw, err := json.Marshal(p) - if err != nil { - return 0, err +func (s kvState) Get(key string) ([]byte, uint64, error) { + e, err := s.kv.Get(s.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil } - rev, err := s.kv.Update(s.ctx, key, raw, revision) + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (s kvState) Create(key string, value []byte) (uint64, error) { + rev, err := s.kv.Create(s.ctx, key, value) + if errors.Is(err, jetstream.ErrKeyExists) { + return 0, pipeline.ErrExists + } + return rev, err +} + +func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) { + rev, err := s.kv.Update(s.ctx, key, value, revision) if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) { return 0, pipeline.ErrChanged } return rev, err } -func (s pipelineStore) Keys(filter string) ([]string, error) { - all, err := s.kv.Keys(s.ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil +func (s kvState) Delete(key string) error { + if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err } + return nil +} + +// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects. +func (s kvState) Keys(filter string) ([]string, error) { + l, err := s.kv.ListKeysFiltered(s.ctx, filter) + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return nil, err + } + // The lister ends its own watch once the current keys are listed. var out []string - want := strings.Split(filter, ".") - for _, k := range all { - got := strings.Split(k, ".") - if len(got) != len(want) { - continue - } - ok := true - for i := range want { - ok = ok && (want[i] == "*" || want[i] == got[i]) - } - if ok { - out = append(out, k) - } + for k := range l.Keys() { + out = append(out, k) } - return out, err + return out, nil } // newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its // objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly. -func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue, +func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue, say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer { - return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module, + return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module, Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append, Keep: func(key string, k seat.KeptResult) error { raw, err := json.Marshal(k) diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go new file mode 100644 index 00000000..9b7f1e07 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys.go @@ -0,0 +1,396 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its +// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under +// `steps..`, written by compare-and-set as the record is, so a record never grows with the catalog and no +// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names. + +// StepsKey is the key of one mover's steps in the state pipelines, `steps..`, the mover's module escaped +// as a key's name is (H-A-187). +func StepsKey(pipeline, mover string) string { + return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover)) +} + +// stepsPrefix is the start of every mover key of one pipeline. +func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") } + +// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a +// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover. +func IsStepsKey(key string) bool { + parts := strings.Split(key, ".") + if parts[0] != "steps" { + return false + } + switch len(parts) { + case 4, 7: + return true + case 5: + return parts[2] == runToken + } + return false +} + +// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by +// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a +// filter, `*` one key part. +type RawState interface { + Get(key string) ([]byte, uint64, error) + Create(key string, value []byte) (uint64, error) + Update(key string, value []byte, revision uint64) (uint64, error) + Delete(key string) error + Keys(filter string) ([]string, error) +} + +// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold. +var ErrNoPipeline = errors.New("no such pipeline") + +// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its +// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the +// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one +// pipeline take its lock, so the process never reads a record and mover keys of two writes. +type MoverStore struct { + Raw RawState + Logf func(string, ...any) + + mu sync.Mutex + locks map[string]*sync.Mutex + read map[string]readAt + inline map[string]bool +} + +// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision. +type readAt struct { + rev uint64 + movers map[string]uint64 +} + +func (s *MoverStore) lock(key string) func() { + s.mu.Lock() + if s.locks == nil { + s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{} + } + l, ok := s.locks[key] + if !ok { + l = &sync.Mutex{} + s.locks[key] = l + } + s.mu.Unlock() + l.Lock() + return l.Unlock +} + +func (s *MoverStore) logf(f string, a ...any) { + if s.Logf != nil { + s.Logf(f, a...) + } +} + +// moverKeys are the mover keys of pipeline key with their values and revisions. +func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) { + ks, err := s.Raw.Keys(stepsPrefix(key) + "*") + if err != nil { + return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err) + } + vals, revs := map[string][]byte{}, map[string]uint64{} + for _, k := range ks { + v, rev, err := s.Raw.Get(k) + if err != nil { + return nil, nil, fmt.Errorf("%s is not read: %w", k, err) + } + if v != nil { + vals[k], revs[k] = v, rev + } + } + return vals, revs, nil +} + +// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order, and +// its unlogged step-log entries: the record's own (an entry naming no mover, written before H-A-190) and each mover's, in +// the order they ended (H-A-189). +func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error { + var names []string + for k := range vals { + names = append(names, k) + } + sort.Strings(names) + order := make([]string, 0, len(names)) + for _, m := range p.Movers { + if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) { + order = append(order, k) + } + } + for _, k := range names { + if !slices.Contains(order, k) { + order = append(order, k) + } + } + steps := []seat.StepResult{} + unlogged := append([]seat.LogEntry(nil), p.Unlogged...) + for _, k := range order { + v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k]) + if err != nil { + return fmt.Errorf("%s holds no mover's steps: %w", k, err) + } + steps = append(steps, v.Steps...) + unlogged = append(unlogged, v.Unlogged...) + } + sort.SliceStable(unlogged, func(i, j int) bool { return unlogged[i].At.Before(unlogged[j].At) }) + p.Steps, p.Unlogged = steps, unlogged + return nil +} + +// split is p's record, its own steps and unlogged entries empty, and each mover's steps and unlogged step-log entries by +// its mover key (H-A-187, H-A-189). An entry naming no mover (one written before H-A-190) goes under the mover whose step +// it alone matches, else it stays in the record. +func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) { + by := map[string]*seat.PipelinesByStepsPipelineMoverValue{} + at := func(module string) *seat.PipelinesByStepsPipelineMoverValue { + k := StepsKey(key, module) + if by[k] == nil { + by[k] = &seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{}} + } + return by[k] + } + for _, st := range p.Steps { + if st.Module == "" { + return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key) + } + v := at(st.Module) + v.Steps = append(v.Steps, st) + } + var kept []seat.LogEntry + for _, e := range p.Unlogged { + module := e.Module + if module == "" { + module = moverOfEntry(p, e) + } + if module == "" { + kept = append(kept, e) + continue + } + v := at(module) + v.Unlogged = append(v.Unlogged, e) + } + movers := map[string][]byte{} + for k, v := range by { + raw, err := json.Marshal(v) + if err != nil { + return nil, nil, err + } + movers[k] = raw + } + p.Steps, p.Unlogged = []seat.StepResult{}, kept + record, err := json.Marshal(p) + return record, movers, err +} + +// moverOfEntry is the mover of the one step an entry naming no mover matches by step and artifact, empty when none or +// more than one does. +func moverOfEntry(p seat.Pipeline, e seat.LogEntry) string { + found := "" + for _, st := range p.Steps { + if st.Step != e.Step || st.Artifact != e.Artifact { + continue + } + if found != "" && found != st.Module { + return "" + } + found = st.Module + } + return found +} + +// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one +// written before H-A-187) is read whole, said once in the journal, and split at its next write. +func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) { + defer s.lock(key)() + raw, rev, err := s.Raw.Get(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + if raw == nil { + return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + var p seat.Pipeline + if err := json.Unmarshal(raw, &p); err != nil { + return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return seat.Pipeline{}, 0, err + } + s.read[key] = readAt{rev: rev, movers: revs} + if len(p.Steps) > 0 { + if !s.inline[key] { + s.inline[key] = true + s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps)) + } + return p, rev, nil + } + if p.Steps == nil { + p.Steps = []seat.StepResult{} + } + return p, rev, composeSteps(&p, key, vals) +} + +// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged, +// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its +// mover keys written before the record, so its steps are never lost between the two. +func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) { + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return 0, err + } + raw, cur, err := s.Raw.Get(key) + if err != nil { + return 0, err + } + if raw == nil { + return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key) + } + if cur != revision { + return 0, ErrChanged + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return 0, err + } + // A mover key written since the read is another write between: the read named its revision. + if seen, ok := s.read[key]; ok && seen.rev == revision { + for k, r := range revs { + if seen.movers[k] != r { + return 0, ErrChanged + } + } + for k := range seen.movers { + if _, ok := revs[k]; !ok { + return 0, ErrChanged + } + } + } + var was seat.Pipeline + inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0 + var after map[string]uint64 + if inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } + rev, err := s.Raw.Update(key, record, revision) + if err != nil { + return 0, err + } + if !inline { + if after, err = s.writeMovers(key, want, vals, revs); err != nil { + return 0, err + } + } else { + s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want)) + delete(s.inline, key) + } + s.read[key] = readAt{rev: rev, movers: after} + return rev, nil +} + +// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the +// revisions read, and answers every mover key's revision after it. +func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) { + after := map[string]uint64{} + for _, k := range slices.Sorted(maps.Keys(want)) { + v := want[k] + old, ok := vals[k] + var rev uint64 + var err error + switch { + case ok && string(old) == string(v): + after[k] = revs[k] + continue + case ok: + rev, err = s.Raw.Update(k, v, revs[k]) + default: + rev, err = s.Raw.Create(k, v) + } + if err != nil { + return nil, fmt.Errorf("the steps of %s are not written: %w", k, err) + } + after[k] = rev + } + for k := range vals { + if _, ok := want[k]; !ok { + if err := s.Raw.Delete(k); err != nil { + return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err) + } + } + } + return after, nil +} + +// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing, +// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal. +func (s *MoverStore) Create(key string, value []byte) error { + var p seat.Pipeline + if err := json.Unmarshal(value, &p); err != nil { + return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err) + } + defer s.lock(key)() + record, want, err := split(key, p) + if err != nil { + return err + } + rev, err := s.Raw.Create(key, record) + if err != nil { + return err + } + vals, revs, err := s.moverKeys(key) + if err != nil { + return err + } + for k := range vals { + s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k) + } + after, err := s.writeMovers(key, want, vals, revs) + if err != nil { + return err + } + s.read[key] = readAt{rev: rev, movers: after} + return nil +} + +// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the +// next ageing deletes again. +func (s *MoverStore) Delete(key string) error { + unlock := s.lock(key) + vals, _, err := s.moverKeys(key) + if err == nil { + for k := range vals { + if err = s.Raw.Delete(k); err != nil { + break + } + } + } + if err == nil { + err = s.Raw.Delete(key) + } + delete(s.read, key) + delete(s.inline, key) + unlock() + return err +} + +// Keys are the keys of the state pipelines matching filter, `*` one key part. +func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go new file mode 100644 index 00000000..b8ea7f87 --- /dev/null +++ b/modules/mesh-delivery-pipeline/internal/pipeline/moverkeys_test.go @@ -0,0 +1,254 @@ +package pipeline + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has. +type memRaw struct { + vals map[string][]byte + revs map[string]uint64 + seq uint64 +} + +func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} } + +func (m *memRaw) Get(key string) ([]byte, uint64, error) { + v, ok := m.vals[key] + if !ok { + return nil, 0, nil + } + return append([]byte(nil), v...), m.revs[key], nil +} + +func (m *memRaw) put(key string, value []byte) uint64 { + m.seq++ + m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq + return m.seq +} + +func (m *memRaw) Create(key string, value []byte) (uint64, error) { + if _, ok := m.vals[key]; ok { + return 0, ErrExists + } + return m.put(key, value), nil +} + +func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) { + if m.revs[key] != rev { + return 0, ErrChanged + } + return m.put(key, value), nil +} + +func (m *memRaw) Delete(key string) error { + delete(m.vals, key) + delete(m.revs, key) + return nil +} + +func (m *memRaw) Keys(filter string) ([]string, error) { + var out []string + for k := range m.vals { + if ok, _ := matchKey(filter, k); ok { + out = append(out, k) + } + } + return out, nil +} + +// recordSteps is the steps the raw record of key carries itself. +func recordSteps(t *testing.T, m *memRaw, key string) int { + t.Helper() + var p seat.Pipeline + if err := json.Unmarshal(m.vals[key], &p); err != nil { + t.Fatal(err) + } + return len(p.Steps) +} + +// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's +// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline +// read whole and split at its next write, one row per case. +func TestMoverKeys(t *testing.T) { + const key = "main.7" + two := func() seat.Pipeline { + return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning, + Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}}, + Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning}, + {Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}} + } + created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + if err := s.Create(key, raw); err != nil { + t.Fatal(err) + } + return m, s, logs + } + steps := func(p seat.Pipeline) string { + var out []string + for _, st := range p.Steps { + out = append(out, st.Module+" "+st.Step+" "+st.Outcome) + } + return strings.Join(out, "; ") + } + rows := []struct { + name string + run func(t *testing.T) string + want string + }{ + {"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string { + m, s, _ := created(t) + p, _, err := s.Get(key) + return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil, + m.vals["steps.main.7.beta"] != nil, steps(p), err) + }, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running "}, + {"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + beta := m.revs["steps.main.7.beta"] + p.Steps[2].Outcome = seat.StepResultOutcomePassed + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err) + }, "beta kept true: beta build running; beta approval not asked; alpha build passed "}, + {"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"]) + p.Outcome = seat.PipelineOutcomePassed + _, err := s.Update(key, p, rev) + return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev) + }, "true, record at true"}, + {"a record written since the read refuses the update", func(t *testing.T) string { + _, s, _ := created(t) + p, rev, _ := s.Get(key) + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + _, err := s.Update(key, p, rev) + return fmt.Sprint(errors.Is(err, ErrChanged)) + }, "true"}, + {"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string { + m, logs := newMemRaw(), &[]string{} + s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }} + raw, _ := json.Marshal(two()) + m.put(key, raw) + p, rev, _ := s.Get(key) + _, _, _ = s.Get(key) + read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs)) + _, err := s.Update(key, p, rev) + q, _, _ := s.Get(key) + return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q), + strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err) + }, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true "}, + {"a second create writes nothing", func(t *testing.T) string { + m, s, _ := created(t) + p := two() + p.Steps[0].Outcome = seat.StepResultOutcomeFailed + raw, _ := json.Marshal(p) + err := s.Create(key, raw) + q, _, _ := s.Get(key) + return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q)) + }, "true 3: beta build running; beta approval not asked; alpha build running"}, + {"a delete removes the mover keys and the record", func(t *testing.T) string { + m, s, _ := created(t) + err := s.Delete(key) + return fmt.Sprintf("%d %v", len(m.vals), err) + }, "0 "}, + {"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + p.Movers, p.Steps = nil, nil + for i := range 129 { + mod := fmt.Sprintf("module-%03d", i) + p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod}) + for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit, + seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition, + seat.StepResultStepReview, seat.StepResultStepApproval} { + p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program", + Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)}) + } + } + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + largest := 0 + for k, v := range m.vals { + if IsStepsKey(k) && len(v) > largest { + largest = len(v) + } + } + q, _, _ := s.Get(key) + return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10, + largest < 16<<10, len(q.Steps)) + }, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"}, + {"a mover's unlogged entries live under its mover key, each filed by the mover it names (H-A-189, H-A-190)", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC) + // Two movers' approvals end alike at one moment; each entry names its mover. + p.Unlogged = []seat.LogEntry{{Pipeline: key, Module: "beta", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at}, + {Pipeline: key, Module: "alpha", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at.Add(time.Second)}} + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + var rec seat.Pipeline + _ = json.Unmarshal(m.vals[key], &rec) + beta, _ := seat.DecodePipelinesByStepsPipelineMoverValue(m.vals["steps.main.7.beta"]) + q, _, _ := s.Get(key) + var got []string + for _, e := range q.Unlogged { + got = append(got, e.Module) + } + return fmt.Sprintf("record %d unlogged, beta's key %d (%s), read %s", len(rec.Unlogged), len(beta.Unlogged), + beta.Unlogged[0].Module, strings.Join(got, " ")) + }, "record 0 unlogged, beta's key 1 (beta), read beta alpha"}, + {"a 129-mover pipeline with 1,000 unlogged entries keeps its record small (H-A-189)", func(t *testing.T) string { + m, s, _ := created(t) + p, rev, _ := s.Get(key) + p.Movers, p.Steps, p.Unlogged = nil, nil, nil + at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC) + for i := range 129 { + mod := fmt.Sprintf("module-%03d", i) + p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod}) + p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeNotAsked}) + } + for i := range 1000 { + mod := fmt.Sprintf("module-%03d", i%129) + p.Unlogged = append(p.Unlogged, seat.LogEntry{Pipeline: key, Module: mod, Step: seat.StepResultStepBuild, Artifact: mod + ":program", + What: seat.LogEntryWhatEnded, Outcome: seat.StepResultOutcomeNotAsked, Output: strings.Repeat("o", 200), At: at.Add(time.Duration(i) * time.Millisecond)}) + } + if _, err := s.Update(key, p, rev); err != nil { + return err.Error() + } + q, _, _ := s.Get(key) + return fmt.Sprintf("record under 64 KiB %t, %d unlogged read", len(m.vals[key]) < 64<<10, len(q.Unlogged)) + }, "record under 64 KiB true, 1000 unlogged read"}, + {"mover keys are told from records of every key form", func(t *testing.T) string { + var out []string + for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha", + "main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} { + out = append(out, fmt.Sprint(IsStepsKey(k))) + } + return strings.Join(out, " ") + }, "true true true false false false false"}, + } + for _, r := range rows { + t.Run(r.name, func(t *testing.T) { + if got := r.run(t); got != r.want { + t.Errorf("got %s\nwant %s", got, r.want) + } + }) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read.go b/modules/mesh-delivery-pipeline/internal/pipeline/read.go index bdf25787..e6a0762b 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read.go @@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) { return f, nil } -// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83). +// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the +// pipeline's record with that mover's steps alone (H-A-187). func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { p, ok, err := r.pipeline(in.Pipeline) if err != nil { @@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { if err != nil { return seat.ShowAnswer{}, err } + // Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers. + if m := strings.TrimSpace(in.Mover); m != "" { + if !Moves(p, m) { + return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m) + } + p.Steps = stepsOf(p, m) + } a := seat.ShowAnswer{Pipeline: p, Fill: fills} for _, s := range p.Steps { if s.Output != "" { @@ -62,7 +70,7 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) { } // List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it -// names one (§83). +// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191). func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) { all, err := r.pipelines() if err != nil { @@ -72,6 +80,9 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) { for _, p := range all { if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) && (in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) { + // Each pipeline's record with its movers and steps counted, never its steps: show gives those (H-A-191). + p.MoverCount, p.StepCount = int64(len(p.Movers)), int64(len(p.Steps)) + p.Steps, p.Unlogged = []seat.StepResult{}, nil out = append(out, p) } } @@ -178,6 +189,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if key == "" { return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, ..") } + all, err := r.Pipelines.Keys() + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err) + } + return r.pipelineOf(key, all) +} + +// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that +// still carries its steps inline is read whole. +func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) { raw, err := r.Pipelines.Value(key) if err != nil { return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err) @@ -192,7 +213,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) { if p.Pipeline == "" { p.Pipeline = key } - return p, true, nil + if len(p.Steps) > 0 { + return p, true, nil + } + prefix := stepsPrefix(key) + vals := map[string][]byte{} + for _, k := range all { + if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") { + continue + } + v, err := r.Pipelines.Value(k) + if err != nil { + return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err) + } + if v != nil { + vals[k] = v + } + } + return p, true, composeSteps(&p, key, vals) } func (r Reads) pipelines() ([]seat.Pipeline, error) { @@ -203,7 +241,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) { sort.Strings(keys) var out []seat.Pipeline for _, k := range keys { - p, ok, err := r.pipeline(k) + if IsStepsKey(k) { + continue + } + p, ok, err := r.pipelineOf(k, keys) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go index ca728003..375d0289 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/read_test.go @@ -3,6 +3,7 @@ package pipeline import ( "encoding/json" "errors" + "fmt" "strings" "testing" "time" @@ -49,6 +50,15 @@ func TestReads(t *testing.T) { Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}}) put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main", Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}}) + // A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps... + put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning, + Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}}) + put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon", + Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}}) + put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta", + Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed}, {Module: "delta", + Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}}, + Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}}) put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"}) put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"}) r := Reads{Pipelines: pipes, Kept: kept} @@ -73,6 +83,35 @@ func TestReads(t *testing.T) { _, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"}) return "", err }, `error: no pipeline "main.9.alpha" in pipelines`}, + {"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err + }, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"}, + {"show naming a mover answers its steps alone", func() (string, error) { + a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"}) + var s []string + for _, st := range a.Pipeline.Steps { + s = append(s, st.Module+" "+st.Step) + } + return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err + }, "epsilon build, 2 movers"}, + {"show refuses a mover the pipeline does not move", func() (string, error) { + _, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"}) + return "", err + }, "error: the pipeline next.8 does not move alpha"}, + {"list names a pipeline, never a mover key, its record with its movers and steps counted and no step (H-A-191)", func() (string, error) { + a, err := r.List(seat.ListInput{Branch: "next"}) + var s []string + for _, p := range a.Pipelines { + s = append(s, fmt.Sprintf("%s %d movers %d steps, %d steps and %d unlogged answered", p.Pipeline, p.MoverCount, p.StepCount, + len(p.Steps), len(p.Unlogged))) + } + return strings.Join(s, "; "), err + }, "next.8 2 movers 3 steps, 0 steps and 0 unlogged answered"}, {"list by module, newest first", func() (string, error) { a, err := r.List(seat.ListInput{Module: "alpha"}) return names(a.Pipelines), err @@ -92,7 +131,7 @@ func TestReads(t *testing.T) { s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04")) } return strings.Join(s, ";"), err - }, "main.7.alpha approval person 10:01"}, + }, "main.7.alpha approval person 10:01;next.8 approval person 10:02"}, {"why-carried names where a carried step came from", func() (string, error) { a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"}) return a.CarriedFrom, err diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go index ecb6ccdc..9b39cf26 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/steps.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/steps.go @@ -441,7 +441,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error { // endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried. func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry { - return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome, + return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome, Output: s.Output, Kept: kept, At: at} } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/unlogged.go b/modules/mesh-delivery-pipeline/internal/pipeline/unlogged.go index e138eefe..fd9211f9 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/unlogged.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/unlogged.go @@ -7,9 +7,10 @@ import ( seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) -// sameEntry says whether two step-log entries are one: the same pipeline, step, artifact, end or start and time. +// sameEntry says whether two step-log entries are one: the same pipeline, mover (H-A-190), step, artifact, end or start and +// time. func sameEntry(a, b seat.LogEntry) bool { - return a.Pipeline == b.Pipeline && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At) + return a.Pipeline == b.Pipeline && a.Module == b.Module && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At) } // appendUnlogged appends the unlogged entries of pipeline key to the step log in their order and clears each appended one in a