Design 47 H1: a pipeline's steps and unlogged entries live under one key per mover (H-A-187, H-A-189, H-A-190, H-A-191) #292
@@ -276,6 +276,48 @@ func DecodePipelinesByBranchRunRunValue(raw []byte) (PipelinesByBranchRunRunValu
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
|
||||
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
|
||||
return "steps." + pipeline + "." + mover
|
||||
}
|
||||
|
||||
// ParsePipelinesByStepsPipelineMoverKey reads the parts of a key of the form "steps.<pipeline>.<mover>", in the order of the form; the last result is false for a key of another form.
|
||||
func ParsePipelinesByStepsPipelineMoverKey(key string) (string, string, bool) {
|
||||
rest := key
|
||||
if !strings.HasPrefix(rest, "steps.") {
|
||||
return "", "", false
|
||||
}
|
||||
rest = rest[6:]
|
||||
v1, after1, found1 := strings.Cut(rest, ".")
|
||||
if !found1 || v1 == "" {
|
||||
return "", "", false
|
||||
}
|
||||
rest = after1
|
||||
if rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
v3 := rest
|
||||
rest = ""
|
||||
if rest != "" {
|
||||
return "", "", false
|
||||
}
|
||||
return v1, v3, true
|
||||
}
|
||||
|
||||
// DecodePipelinesByStepsPipelineMoverValue reads one value of the state pipelines under the form "steps.<pipeline>.<mover>", refusing a field PipelinesByStepsPipelineMoverValue does not know and any text after the value.
|
||||
func DecodePipelinesByStepsPipelineMoverValue(raw []byte) (PipelinesByStepsPipelineMoverValue, error) {
|
||||
var v PipelinesByStepsPipelineMoverValue
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return v, err
|
||||
}
|
||||
if dec.More() {
|
||||
return v, errors.New("text after the value")
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// QueueByQueueNextKey is the key of one value of the state queue under the form "queue.next".
|
||||
func QueueByQueueNextKey() string { return "queue.next" }
|
||||
|
||||
@@ -523,8 +565,10 @@ type LogEntry struct {
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470)
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
// the module of the mover whose step the entry is of (H-A-190)
|
||||
Module string `json:"module,omitempty"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
// the output's name in step-outputs
|
||||
Output string `json:"output,omitempty"`
|
||||
Pipeline string `json:"pipeline"`
|
||||
@@ -621,6 +665,8 @@ type Pipeline struct {
|
||||
// how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325)
|
||||
EndCount int64 `json:"end-count,omitempty"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
// how many movers the pipeline has, set by list alone (H-A-191)
|
||||
MoverCount int64 `json:"mover-count,omitempty"`
|
||||
// the modules the pipeline moves, its take-in's entries (design 47 H §746)
|
||||
Movers []Mover `json:"movers"`
|
||||
Outcome string `json:"outcome"`
|
||||
@@ -637,11 +683,13 @@ type Pipeline struct {
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
// every mover's steps, each naming its mover, approval last
|
||||
// how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)
|
||||
StepCount int64 `json:"step-count,omitempty"`
|
||||
// every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)
|
||||
Steps []StepResult `json:"steps"`
|
||||
// the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107)
|
||||
Superseded []string `json:"superseded,omitempty"`
|
||||
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)
|
||||
// the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)
|
||||
Unlogged []LogEntry `json:"unlogged,omitempty"`
|
||||
// the layout version
|
||||
V int64 `json:"v,omitempty"`
|
||||
@@ -1026,6 +1074,8 @@ type StartAnswer struct {
|
||||
|
||||
// ShowInput is what show takes.
|
||||
type ShowInput struct {
|
||||
// a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)
|
||||
Mover string `json:"mover,omitempty"`
|
||||
// the pipeline's key
|
||||
Pipeline string `json:"pipeline"`
|
||||
}
|
||||
@@ -1314,6 +1364,14 @@ type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline
|
||||
// PipelinesByBranchRunRunValue is one value of the state pipelines under the key <branch>.run.<run>.
|
||||
type PipelinesByBranchRunRunValue = Pipeline
|
||||
|
||||
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
|
||||
type PipelinesByStepsPipelineMoverValue struct {
|
||||
// the mover's steps
|
||||
Steps []StepResult `json:"steps"`
|
||||
// the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)
|
||||
Unlogged []LogEntry `json:"unlogged,omitempty"`
|
||||
}
|
||||
|
||||
// QueueByQueueNextValue is one value of the state queue under the key queue.next.
|
||||
type QueueByQueueNextValue = int64
|
||||
|
||||
|
||||
@@ -86,13 +86,17 @@
|
||||
},
|
||||
{
|
||||
"name": "show",
|
||||
"description": "One pipeline whole: its steps, its step log and its outputs, with the stores' fill. Reads only.",
|
||||
"description": "One pipeline whole: its record with every mover's steps, or one mover's steps alone when it names the mover, its step log and its outputs, with the stores' fill. Reads only.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"pipeline": {
|
||||
"type": "string",
|
||||
"description": "the pipeline's key"
|
||||
},
|
||||
"mover": {
|
||||
"type": "string",
|
||||
"description": "a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -133,7 +137,7 @@
|
||||
},
|
||||
{
|
||||
"name": "list",
|
||||
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill. Reads only.",
|
||||
"description": "The pipelines and runs of a module, a branch or a commit, newest first, with the store's fill: each pipeline's record with its movers and steps counted, never its steps (show gives those). Reads only.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -771,7 +775,7 @@
|
||||
"name": "pipelines",
|
||||
"template": "state",
|
||||
"history": 1,
|
||||
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747)",
|
||||
"description": "one key per pipeline, one per take-in, written by compare-and-set on its absence (design 47 H §746, §747), and one key per mover of a pipeline holding that mover's steps (H-A-187)",
|
||||
"key-forms": [
|
||||
{
|
||||
"key": "<branch>.<source>",
|
||||
@@ -793,6 +797,32 @@
|
||||
"$ref": "#/defs/Pipeline"
|
||||
},
|
||||
"description": "a run on request's pipeline, one per run, its movers the named commits' modules; it delivers nothing (H-A-83)"
|
||||
},
|
||||
{
|
||||
"key": "steps.<pipeline>.<mover>",
|
||||
"value": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/StepResult"
|
||||
},
|
||||
"description": "the mover's steps"
|
||||
},
|
||||
"unlogged": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/defs/LogEntry"
|
||||
},
|
||||
"description": "the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"steps"
|
||||
]
|
||||
},
|
||||
"description": "one mover's steps and unlogged step-log entries in a pipeline, its key the pipeline's and the mover's module escaped as a key's name, written by compare-and-set as the pipeline's record is; the record's own steps stay empty, so a record never grows with the catalog (H-A-187)"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1238,7 +1268,7 @@
|
||||
"items": {
|
||||
"$ref": "#/defs/StepResult"
|
||||
},
|
||||
"description": "every mover's steps, each naming its mover, approval last"
|
||||
"description": "every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187)"
|
||||
},
|
||||
"started": {
|
||||
"type": "string",
|
||||
@@ -1253,7 +1283,15 @@
|
||||
"items": {
|
||||
"$ref": "#/defs/LogEntry"
|
||||
},
|
||||
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606)"
|
||||
"description": "the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189)"
|
||||
},
|
||||
"mover-count": {
|
||||
"type": "integer",
|
||||
"description": "how many movers the pipeline has, set by list alone (H-A-191)"
|
||||
},
|
||||
"step-count": {
|
||||
"type": "integer",
|
||||
"description": "how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191)"
|
||||
},
|
||||
"published": {
|
||||
"$ref": "#/defs/Published",
|
||||
@@ -1613,6 +1651,10 @@
|
||||
"pipeline": {
|
||||
"type": "string"
|
||||
},
|
||||
"module": {
|
||||
"type": "string",
|
||||
"description": "the module of the mover whose step the entry is of (H-A-190)"
|
||||
},
|
||||
"step": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
|
||||
@@ -112,6 +112,12 @@ export declare function pipelinesByBranchRunRunKey(branch: string, run: string):
|
||||
/** The parts of a key of the form "<branch>.run.<run>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRunRunKey(_key: string): { branch: string; run: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
|
||||
|
||||
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByStepsPipelineMoverKey(_key: string): { pipeline: string; mover: string } | undefined;
|
||||
|
||||
/** The key of one value of the state queue under the form "queue.next". */
|
||||
export declare function queueByQueueNextKey(): string;
|
||||
|
||||
@@ -186,6 +192,8 @@ export interface LogEntry {
|
||||
"at": string;
|
||||
/** the keys of the kept results the step wrote or carried, <level>.<digest>, which the runner ages by their last-read (design 47 G sections 330, 470) */
|
||||
"kept"?: string[];
|
||||
/** the module of the mover whose step the entry is of (H-A-190) */
|
||||
"module"?: string;
|
||||
"outcome"?: "running" | "waiting" | "passed" | "failed" | "nothing to run" | "bypassed" | "carried" | "could not run" | "waiting for a build node" | "did not build" | "accepted" | "declined" | "superseded" | "cancelled";
|
||||
/** the output's name in step-outputs */
|
||||
"output"?: string;
|
||||
@@ -248,6 +256,8 @@ export interface Pipeline {
|
||||
/** how often the pipeline has ended: 1 at its first end, one higher at each end again after a passing rerun, so the assigner takes a later end of the same source revision (design 47 G sections 29, 325) */
|
||||
"end-count"?: number;
|
||||
"ended"?: string;
|
||||
/** how many movers the pipeline has, set by list alone (H-A-191) */
|
||||
"mover-count"?: number;
|
||||
/** the modules the pipeline moves, its take-in's entries (design 47 H §746) */
|
||||
"movers": Mover[];
|
||||
"outcome": "waiting" | "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
|
||||
@@ -264,11 +274,13 @@ export interface Pipeline {
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source": number;
|
||||
"started"?: string;
|
||||
/** every mover's steps, each naming its mover, approval last */
|
||||
/** how many steps the pipeline's movers have, set by list alone, which answers the record with its steps empty (H-A-191) */
|
||||
"step-count"?: number;
|
||||
/** every mover's steps, each naming its mover, approval last; in the state pipelines a record's own are empty, each mover's kept under its own key (H-A-187) */
|
||||
"steps": StepResult[];
|
||||
/** the modules a newer waiting take-in of the same branch took out of the pipeline, written with the movers by compare-and-set; at its placing the runner writes a mover's build-step why from it (design 47 H §748, H-A-107) */
|
||||
"superseded"?: string[];
|
||||
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606) */
|
||||
/** the step-log entries of the steps that ended and are not appended to the step log yet, in order; written with the step's end in one write and cleared when appended (design 47 G sections 486, 606); in the state pipelines a record's own are empty, each mover's kept under its own key beside its steps (H-A-189) */
|
||||
"unlogged"?: LogEntry[];
|
||||
/** the layout version */
|
||||
"v"?: number;
|
||||
@@ -556,6 +568,8 @@ export interface StartAnswer {
|
||||
|
||||
/** is what show takes. */
|
||||
export interface ShowInput {
|
||||
/** a mover's module: the answer's pipeline then holds that mover's steps alone (H-A-187) */
|
||||
"mover"?: string;
|
||||
/** the pipeline's key */
|
||||
"pipeline": string;
|
||||
}
|
||||
@@ -795,6 +809,14 @@ export type PipelinesByBranchSourcePublishedPackageVersionValue = Pipeline;
|
||||
/** is one value of the state pipelines under the key <branch>.run.<run>. */
|
||||
export type PipelinesByBranchRunRunValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
|
||||
export interface PipelinesByStepsPipelineMoverValue {
|
||||
/** the mover's steps */
|
||||
"steps": StepResult[];
|
||||
/** the step-log entries of the mover's steps that ended and are not appended to the step log yet, in order (H-A-189) */
|
||||
"unlogged"?: LogEntry[];
|
||||
}
|
||||
|
||||
/** is one value of the state queue under the key queue.next. */
|
||||
export type QueueByQueueNextValue = number;
|
||||
|
||||
|
||||
@@ -173,6 +173,27 @@ export function parsePipelinesByBranchRunRunKey(_key) {
|
||||
return { branch, run };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
|
||||
return "steps." + pipeline + "." + mover;
|
||||
}
|
||||
|
||||
/** The parts of a key of the form "steps.<pipeline>.<mover>", or undefined for a key of another form. */
|
||||
export function parsePipelinesByStepsPipelineMoverKey(_key) {
|
||||
let _rest = _key;
|
||||
if (!_rest.startsWith("steps.")) return undefined;
|
||||
_rest = _rest.slice(6);
|
||||
const _at1 = _rest.indexOf(".");
|
||||
if (_at1 <= 0) return undefined;
|
||||
const pipeline = _rest.slice(0, _at1);
|
||||
_rest = _rest.slice(_at1 + 1);
|
||||
if (_rest === "") return undefined;
|
||||
const mover = _rest;
|
||||
_rest = "";
|
||||
if (_rest !== "") return undefined;
|
||||
return { pipeline, mover };
|
||||
}
|
||||
|
||||
/** The key of one value of the state queue under the form "queue.next". */
|
||||
export function queueByQueueNextKey() {
|
||||
return "queue.next";
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
// §549): runner_unlogged lists them, runner_unlogged_append appends them by hand, written in the hand-act log; it is of class
|
||||
// act and takes §391's default callers.
|
||||
func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.HandActs, logf func(string, ...any)) []stdio.Tool {
|
||||
h := &pipeline.Hearer{Store: stdioPipelines{pipelines}, Logf: logf,
|
||||
h := &pipeline.Hearer{Store: &pipeline.MoverStore{Raw: stdioPipelines{pipelines}, Logf: logf}, Logf: logf,
|
||||
Log: func(e seat.LogEntry) error {
|
||||
_, err := log.Append(keys.EscapeName(e.Pipeline), e)
|
||||
return err
|
||||
@@ -67,26 +67,36 @@ func UnloggedTools(pipelines stdio.KeptState, log stdio.KeptLog, acts views.Hand
|
||||
}
|
||||
}
|
||||
|
||||
// stdioPipelines is the state pipelines as the runtime serves it, for the by-hand append.
|
||||
// stdioPipelines is the state pipelines as bytes, as the runtime serves it, for the by-hand append: read and written through
|
||||
// a pipeline.MoverStore, which keeps each mover's steps under its own key (H-A-187).
|
||||
type stdioPipelines struct{ s stdio.KeptState }
|
||||
|
||||
func (s stdioPipelines) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
func (s stdioPipelines) Get(key string) ([]byte, uint64, error) {
|
||||
e, err := s.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return seat.Pipeline{}, 0, errors.Join(err, errors.New("no such pipeline"))
|
||||
return nil, 0, err
|
||||
}
|
||||
var p seat.Pipeline
|
||||
return p, e.Revision, json.Unmarshal(e.Value, &p)
|
||||
return e.Value, e.Revision, nil
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Update(key string, p seat.Pipeline, rev uint64) (uint64, error) {
|
||||
r, err := s.s.Update(key, p, rev)
|
||||
func (s stdioPipelines) Create(key string, value []byte) (uint64, error) {
|
||||
r, err := s.s.Create(key, json.RawMessage(value))
|
||||
if err != nil && strings.Contains(err.Error(), "exists") {
|
||||
return 0, pipeline.ErrExists
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Update(key string, value []byte, rev uint64) (uint64, error) {
|
||||
r, err := s.s.Update(key, json.RawMessage(value), rev)
|
||||
if err != nil && (strings.Contains(err.Error(), "wrong last sequence") || strings.Contains(err.Error(), "revision")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (s stdioPipelines) Delete(key string) error { return s.s.Delete(key) }
|
||||
|
||||
func (s stdioPipelines) Keys(filter string) ([]string, error) {
|
||||
all, err := s.s.Keys()
|
||||
var out []string
|
||||
|
||||
@@ -136,9 +136,9 @@ func (k keptStore) Get(key string) (seat.KeptResult, bool, error) {
|
||||
func (k keptStore) Delete(key string) error { return k.kv.Delete(k.ctx, key) }
|
||||
|
||||
// newAger is the ageing over the runner's own stores.
|
||||
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
|
||||
func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines *pipeline.MoverStore, kept, aged jetstream.KeyValue, logf func(string, ...any)) *pipeline.Ager {
|
||||
return &pipeline.Ager{Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Read, Aged: agedState{ctx: ctx, kv: aged},
|
||||
Pipelines: pipelineDeleter{pipelineStore{ctx: ctx, kv: pipelines}}, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
|
||||
Pipelines: pipelines, Kept: keptStore{ctx: ctx, kv: kept}, Logf: logf,
|
||||
Outputs: func(name string) error {
|
||||
o, err := js.ObjectStore(ctx, catalogue.ObjectsBucket(cred.Module, seat.StateStepOutputs))
|
||||
if err != nil {
|
||||
@@ -151,11 +151,6 @@ func newAger(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
||||
}}
|
||||
}
|
||||
|
||||
// pipelineDeleter is the pipelines store with the delete only its owner makes (§330).
|
||||
type pipelineDeleter struct{ pipelineStore }
|
||||
|
||||
func (d pipelineDeleter) Delete(key string) error { return d.kv.Delete(d.ctx, key) }
|
||||
|
||||
// storeFills reads the fill of the runner's stores against their caps: pipelines and kept (their key-value streams), step-outputs
|
||||
// (its object store's) and step-log (its log's), as the bus's own streams say it (design 47 G §366, §486).
|
||||
func storeFills(ctx context.Context, js jetstream.JetStream, cred process.Account, now func() time.Time) func() ([]seat.Fill, error) {
|
||||
|
||||
@@ -128,9 +128,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return err
|
||||
}
|
||||
hearer := newHearer(ctx, js, cred, pipelines, kept, say, logger.Printf)
|
||||
store := newPipelineStore(ctx, pipelines, logger.Printf)
|
||||
hearer := newHearer(ctx, js, cred, store, kept, say, logger.Printf)
|
||||
hearer.Asker = busAsker{conn: conn, cred: cred}
|
||||
hearer.Ager = newAger(ctx, js, cred, pipelines, kept, aged, logger.Printf)
|
||||
hearer.Ager = newAger(ctx, js, cred, store, kept, aged, logger.Printf)
|
||||
// The stores' fill is said before a store is full, and a full one refuses a start (§366).
|
||||
fills := &pipeline.Fills{Read: storeFills(ctx, js, cred, time.Now), Logf: logger.Printf,
|
||||
Say: func(event string, f seat.Fill) error { return say("mesh.seat."+seat.Seat+".event."+event, f) }}
|
||||
@@ -147,7 +148,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
hearer.Failures = &pipeline.Failures{Findings: rv.Findings, Observer: cred.Module, Author: commitAuthor(calls.Call), Logf: logger.Printf}
|
||||
// Its asking half (§22, §24): a pipeline's steps are placed from the module's registration and each build seat is asked
|
||||
// for its step.
|
||||
starter := &pipeline.Starter{Pipelines: createdKV{ctx: ctx, kv: pipelines}, Logf: logger.Printf, Say: say,
|
||||
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
@@ -170,10 +171,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return
|
||||
}
|
||||
// A pipeline of the branch whose registration was heard before this half was held gets its steps now (A-327).
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited on its half for their steps: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
n, err := starter.Resume(pipelineStore{ctx: ctx, kv: pipelines}, branch, half)
|
||||
n, err := starter.Resume(store, branch, half)
|
||||
if err != nil {
|
||||
logger.Printf("the steps placed on %s and not yet asked: %d asked, then %v", branch, n, err)
|
||||
return
|
||||
@@ -184,19 +185,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view reaching a newer revision places the steps of the pipelines that waited for it, and asks nothing again (A-448, §200).
|
||||
advance := func(branch string) {
|
||||
rv.Holders.refresh()
|
||||
if n, err := starter.Waiting(pipelineStore{ctx: ctx, kv: pipelines}, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Waiting(store, branch, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s that waited for a newer revision of its half: %d placed (%v)", branch, n, err)
|
||||
}
|
||||
}
|
||||
advancer.Store(&advance)
|
||||
// A registration-changed places the steps of its module's pipelines started before it, from it (A-327).
|
||||
register := func(reg catalog.Registration) {
|
||||
if n, err := starter.Registered(pipelineStore{ctx: ctx, kv: pipelines}, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
if n, err := starter.Registered(store, reg, rv.Catalog.View); err != nil || n > 0 {
|
||||
logger.Printf("the pipelines of %s started before its registration: %d placed (%v)", reg.Module, n, err)
|
||||
}
|
||||
}
|
||||
registrar.Store(®ister)
|
||||
starter.Store = pipelineStore{ctx: ctx, kv: pipelines}
|
||||
starter.Store = store
|
||||
// The queue (design 47 H §748): at most pipelines-at-once pipelines have their steps asked, the others wait in order in
|
||||
// the state queue, placed at the start from it and on every end.
|
||||
atOnce, waitingCap, err := queueSettings(getenv)
|
||||
|
||||
@@ -401,6 +401,8 @@ func pipelineKeys(t *testing.T, ctx context.Context, kv jetstream.KeyValue) []st
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A mover's steps are a key of their own (H-A-187), placed with its pipeline: no pipeline of their own.
|
||||
ks = slices.DeleteFunc(ks, pipeline.IsStepsKey)
|
||||
slices.Sort(ks)
|
||||
return ks
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -221,6 +222,34 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
|
||||
waitFor(t, ctx, logs, func() bool { return readPipeline(ctx, pipelines, key, &p) && p.Outcome == seat.PipelineOutcomePassed })
|
||||
|
||||
// H-A-187 over the bus: show composes the pipeline from its record and mover keys, naming a mover its steps alone, and a
|
||||
// record written before it, its steps inline, is read whole and split at its next write.
|
||||
shown, err := pipeline.Reads{Pipelines: kvKept{ctx, pipelines}}.Show(seat.ShowInput{Pipeline: key, Mover: "app"})
|
||||
if err != nil || len(shown.Pipeline.Steps) != len(p.Steps) || shown.Pipeline.Steps[0].Module != "app" {
|
||||
t.Fatalf("show of %s naming app = %d steps (%v), want its %d", key, len(shown.Pipeline.Steps), err, len(p.Steps))
|
||||
}
|
||||
inline := p
|
||||
inline.Pipeline = "main.2"
|
||||
inlineRaw, _ := json.Marshal(inline)
|
||||
if _, err := pipelines.Put(ctx, "main.2", inlineRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := newPipelineStore(ctx, pipelines, func(f string, a ...any) { fmt.Fprintf(logs, f+"\n", a...) })
|
||||
old, oldRev, err := store.Get("main.2")
|
||||
if err != nil || len(old.Steps) != len(p.Steps) {
|
||||
t.Fatalf("the inline record read %d steps (%v), want %d", len(old.Steps), err, len(p.Steps))
|
||||
}
|
||||
if _, err := store.Update("main.2", old, oldRev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var split seat.Pipeline
|
||||
if !readPipeline(ctx, pipelines, "main.2", &split) || len(split.Steps) != len(p.Steps) {
|
||||
t.Fatalf("the record split reads %d steps, want %d", len(split.Steps), len(p.Steps))
|
||||
}
|
||||
if _, err := pipelines.Get(ctx, pipeline.StepsKey("main.2", "app")); err != nil || !strings.Contains(logs.String(), "split into 1 mover keys") {
|
||||
t.Fatalf("the mover key of main.2's app: %v; the journal says the split: %t", err, strings.Contains(logs.String(), "split into 1 mover keys"))
|
||||
}
|
||||
|
||||
// §470, §330: the step log is the log's stream, read onward by sequence the way the ageing reads it.
|
||||
logged, err := stepLog{ctx: ctx, js: js, module: module}.Read(1, 100)
|
||||
if err != nil || len(logged) == 0 || logged[0].Entry.Pipeline != key {
|
||||
@@ -274,13 +303,20 @@ func stepOf(p seat.Pipeline, step string) seat.StepResult {
|
||||
return seat.StepResult{}
|
||||
}
|
||||
|
||||
// readPipeline reads pipeline key as the runner does, its steps composed from its mover keys, and holds its record to
|
||||
// H-A-187: a record the runner wrote carries no steps of its own, so it never grows with the catalog.
|
||||
func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *seat.Pipeline) bool {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
*p = seat.Pipeline{}
|
||||
return json.Unmarshal(e.Value(), p) == nil
|
||||
var record seat.Pipeline
|
||||
if json.Unmarshal(e.Value(), &record) != nil || len(record.Steps) > 0 {
|
||||
panic(fmt.Sprintf("the record of %s carries %d steps of its own; each mover's steps are its own key (H-A-187)", key, len(record.Steps)))
|
||||
}
|
||||
got, _, err := newPipelineStore(ctx, kv, nil).Get(key)
|
||||
*p = got
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -13,20 +11,6 @@ import (
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// createdKV is the state pipelines written by compare-and-set on a key's absence (§54).
|
||||
type createdKV struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (c createdKV) Create(key string, value []byte) error {
|
||||
_, err := c.kv.Create(c.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return pipeline.ErrExists
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// hearPublished hears each package-published (design 47 H §746, §747): in its source's running pipeline, or starting one
|
||||
// pipeline of the package's kind-1 dependents that pipeline does not move. The message is acknowledged once that is
|
||||
// written, on a goroutine of its own, so a slow write holds no other event; the durable consumer delivers a publish heard
|
||||
|
||||
@@ -19,64 +19,74 @@ import (
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// pipelineStore is the state pipelines as the hearer reads and writes it: a write by compare-and-set on the revision read (§27).
|
||||
type pipelineStore struct {
|
||||
// kvState is the state pipelines as bytes on the bus, by compare-and-set (§27); the runner reads and writes pipelines
|
||||
// through one pipeline.MoverStore over it, which keeps each mover's steps under its own key (H-A-187).
|
||||
type kvState struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (s pipelineStore) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
e, err := s.kv.Get(s.ctx, key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(e.Value(), &p); err != nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
return p, e.Revision(), nil
|
||||
// newPipelineStore is the process's one store of the state pipelines.
|
||||
func newPipelineStore(ctx context.Context, kv jetstream.KeyValue, logf func(string, ...any)) *pipeline.MoverStore {
|
||||
return &pipeline.MoverStore{Raw: kvState{ctx: ctx, kv: kv}, Logf: logf}
|
||||
}
|
||||
|
||||
func (s pipelineStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
func (s kvState) Get(key string) ([]byte, uint64, error) {
|
||||
e, err := s.kv.Get(s.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, nil
|
||||
}
|
||||
rev, err := s.kv.Update(s.ctx, key, raw, revision)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return e.Value(), e.Revision(), nil
|
||||
}
|
||||
|
||||
func (s kvState) Create(key string, value []byte) (uint64, error) {
|
||||
rev, err := s.kv.Create(s.ctx, key, value)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, pipeline.ErrExists
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s kvState) Update(key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := s.kv.Update(s.ctx, key, value, revision)
|
||||
if err != nil && (errors.Is(err, jetstream.ErrKeyExists) || strings.Contains(err.Error(), "wrong last sequence")) {
|
||||
return 0, pipeline.ErrChanged
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s pipelineStore) Keys(filter string) ([]string, error) {
|
||||
all, err := s.kv.Keys(s.ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
func (s kvState) Delete(key string) error {
|
||||
if err := s.kv.Delete(s.ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keys are the keys matching filter, `*` one key part, as the bus filters a key-value's subjects.
|
||||
func (s kvState) Keys(filter string) ([]string, error) {
|
||||
l, err := s.kv.ListKeysFiltered(s.ctx, filter)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
// The lister ends its own watch once the current keys are listed.
|
||||
var out []string
|
||||
want := strings.Split(filter, ".")
|
||||
for _, k := range all {
|
||||
got := strings.Split(k, ".")
|
||||
if len(got) != len(want) {
|
||||
continue
|
||||
}
|
||||
ok := true
|
||||
for i := range want {
|
||||
ok = ok && (want[i] == "*" || want[i] == got[i])
|
||||
}
|
||||
if ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, err
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// newHearer is the hearer over the runner's own states: pipelines, kept, the step log on its stream and step-outputs on its
|
||||
// objects (§27). Each store is opened where it is used, so one the grants part has not made yet fails that write loudly.
|
||||
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines, kept jetstream.KeyValue,
|
||||
func newHearer(ctx context.Context, js jetstream.JetStream, cred process.Account, pipelines pipeline.Updater, kept jetstream.KeyValue,
|
||||
say func(string, any) error, logf func(string, ...any)) *pipeline.Hearer {
|
||||
return &pipeline.Hearer{Store: pipelineStore{ctx: ctx, kv: pipelines}, Say: say, Logf: logf, Name: cred.Module,
|
||||
return &pipeline.Hearer{Store: pipelines, Say: say, Logf: logf, Name: cred.Module,
|
||||
Log: stepLog{ctx: ctx, js: js, module: cred.Module}.Append,
|
||||
Keep: func(key string, k seat.KeptResult) error {
|
||||
raw, err := json.Marshal(k)
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// A pipeline's steps live one key per mover (design 47 H-A-187): the state pipelines keeps the pipeline's record (its
|
||||
// movers, tiers, call, source and outcome) under the pipeline's key with its own steps empty, and each mover's steps under
|
||||
// `steps.<pipeline>.<mover>`, written by compare-and-set as the record is, so a record never grows with the catalog and no
|
||||
// value nears the bus's message limit (ADR 0486). Every read of a pipeline composes its steps from the mover keys it names.
|
||||
|
||||
// StepsKey is the key of one mover's steps in the state pipelines, `steps.<pipeline>.<mover>`, the mover's module escaped
|
||||
// as a key's name is (H-A-187).
|
||||
func StepsKey(pipeline, mover string) string {
|
||||
return seat.PipelinesByStepsPipelineMoverKey(pipeline, keys.EscapeName(mover))
|
||||
}
|
||||
|
||||
// stepsPrefix is the start of every mover key of one pipeline.
|
||||
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
|
||||
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
|
||||
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`) or a publish's (7 parts) and one mover.
|
||||
func IsStepsKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
if parts[0] != "steps" {
|
||||
return false
|
||||
}
|
||||
switch len(parts) {
|
||||
case 4, 7:
|
||||
return true
|
||||
case 5:
|
||||
return parts[2] == runToken
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
|
||||
// compare-and-set on absence (ErrExists), an update on the revision read (ErrChanged), a delete, and the keys matching a
|
||||
// filter, `*` one key part.
|
||||
type RawState interface {
|
||||
Get(key string) ([]byte, uint64, error)
|
||||
Create(key string, value []byte) (uint64, error)
|
||||
Update(key string, value []byte, revision uint64) (uint64, error)
|
||||
Delete(key string) error
|
||||
Keys(filter string) ([]string, error)
|
||||
}
|
||||
|
||||
// ErrNoPipeline is a read of a pipeline key the state pipelines does not hold.
|
||||
var ErrNoPipeline = errors.New("no such pipeline")
|
||||
|
||||
// MoverStore is the state pipelines as the runner reads and writes it (§27, H-A-187): Get composes a pipeline from its
|
||||
// record and its mover keys, Update writes the record and each mover key that changed, each by compare-and-set on the
|
||||
// revision read, and nothing when any of them was written since. One MoverStore serves a process: a write and a read of one
|
||||
// pipeline take its lock, so the process never reads a record and mover keys of two writes.
|
||||
type MoverStore struct {
|
||||
Raw RawState
|
||||
Logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
locks map[string]*sync.Mutex
|
||||
read map[string]readAt
|
||||
inline map[string]bool
|
||||
}
|
||||
|
||||
// readAt is what the last read of a pipeline found: its record's revision and each mover key's revision.
|
||||
type readAt struct {
|
||||
rev uint64
|
||||
movers map[string]uint64
|
||||
}
|
||||
|
||||
func (s *MoverStore) lock(key string) func() {
|
||||
s.mu.Lock()
|
||||
if s.locks == nil {
|
||||
s.locks, s.read, s.inline = map[string]*sync.Mutex{}, map[string]readAt{}, map[string]bool{}
|
||||
}
|
||||
l, ok := s.locks[key]
|
||||
if !ok {
|
||||
l = &sync.Mutex{}
|
||||
s.locks[key] = l
|
||||
}
|
||||
s.mu.Unlock()
|
||||
l.Lock()
|
||||
return l.Unlock
|
||||
}
|
||||
|
||||
func (s *MoverStore) logf(f string, a ...any) {
|
||||
if s.Logf != nil {
|
||||
s.Logf(f, a...)
|
||||
}
|
||||
}
|
||||
|
||||
// moverKeys are the mover keys of pipeline key with their values and revisions.
|
||||
func (s *MoverStore) moverKeys(key string) (map[string][]byte, map[string]uint64, error) {
|
||||
ks, err := s.Raw.Keys(stepsPrefix(key) + "*")
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the mover keys of %s are not listed: %w", key, err)
|
||||
}
|
||||
vals, revs := map[string][]byte{}, map[string]uint64{}
|
||||
for _, k := range ks {
|
||||
v, rev, err := s.Raw.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s is not read: %w", k, err)
|
||||
}
|
||||
if v != nil {
|
||||
vals[k], revs[k] = v, rev
|
||||
}
|
||||
}
|
||||
return vals, revs, nil
|
||||
}
|
||||
|
||||
// composeSteps sets p's steps from its mover keys, in the order of p's movers, a key no mover names last in key order, and
|
||||
// its unlogged step-log entries: the record's own (an entry naming no mover, written before H-A-190) and each mover's, in
|
||||
// the order they ended (H-A-189).
|
||||
func composeSteps(p *seat.Pipeline, key string, vals map[string][]byte) error {
|
||||
var names []string
|
||||
for k := range vals {
|
||||
names = append(names, k)
|
||||
}
|
||||
sort.Strings(names)
|
||||
order := make([]string, 0, len(names))
|
||||
for _, m := range p.Movers {
|
||||
if k := StepsKey(key, m.Module); vals[k] != nil && !slices.Contains(order, k) {
|
||||
order = append(order, k)
|
||||
}
|
||||
}
|
||||
for _, k := range names {
|
||||
if !slices.Contains(order, k) {
|
||||
order = append(order, k)
|
||||
}
|
||||
}
|
||||
steps := []seat.StepResult{}
|
||||
unlogged := append([]seat.LogEntry(nil), p.Unlogged...)
|
||||
for _, k := range order {
|
||||
v, err := seat.DecodePipelinesByStepsPipelineMoverValue(vals[k])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s holds no mover's steps: %w", k, err)
|
||||
}
|
||||
steps = append(steps, v.Steps...)
|
||||
unlogged = append(unlogged, v.Unlogged...)
|
||||
}
|
||||
sort.SliceStable(unlogged, func(i, j int) bool { return unlogged[i].At.Before(unlogged[j].At) })
|
||||
p.Steps, p.Unlogged = steps, unlogged
|
||||
return nil
|
||||
}
|
||||
|
||||
// split is p's record, its own steps and unlogged entries empty, and each mover's steps and unlogged step-log entries by
|
||||
// its mover key (H-A-187, H-A-189). An entry naming no mover (one written before H-A-190) goes under the mover whose step
|
||||
// it alone matches, else it stays in the record.
|
||||
func split(key string, p seat.Pipeline) ([]byte, map[string][]byte, error) {
|
||||
by := map[string]*seat.PipelinesByStepsPipelineMoverValue{}
|
||||
at := func(module string) *seat.PipelinesByStepsPipelineMoverValue {
|
||||
k := StepsKey(key, module)
|
||||
if by[k] == nil {
|
||||
by[k] = &seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{}}
|
||||
}
|
||||
return by[k]
|
||||
}
|
||||
for _, st := range p.Steps {
|
||||
if st.Module == "" {
|
||||
return nil, nil, fmt.Errorf("a step %s of %s names no mover, so it has no mover key", st.Step, key)
|
||||
}
|
||||
v := at(st.Module)
|
||||
v.Steps = append(v.Steps, st)
|
||||
}
|
||||
var kept []seat.LogEntry
|
||||
for _, e := range p.Unlogged {
|
||||
module := e.Module
|
||||
if module == "" {
|
||||
module = moverOfEntry(p, e)
|
||||
}
|
||||
if module == "" {
|
||||
kept = append(kept, e)
|
||||
continue
|
||||
}
|
||||
v := at(module)
|
||||
v.Unlogged = append(v.Unlogged, e)
|
||||
}
|
||||
movers := map[string][]byte{}
|
||||
for k, v := range by {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
movers[k] = raw
|
||||
}
|
||||
p.Steps, p.Unlogged = []seat.StepResult{}, kept
|
||||
record, err := json.Marshal(p)
|
||||
return record, movers, err
|
||||
}
|
||||
|
||||
// moverOfEntry is the mover of the one step an entry naming no mover matches by step and artifact, empty when none or
|
||||
// more than one does.
|
||||
func moverOfEntry(p seat.Pipeline, e seat.LogEntry) string {
|
||||
found := ""
|
||||
for _, st := range p.Steps {
|
||||
if st.Step != e.Step || st.Artifact != e.Artifact {
|
||||
continue
|
||||
}
|
||||
if found != "" && found != st.Module {
|
||||
return ""
|
||||
}
|
||||
found = st.Module
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// Get is pipeline key with every mover's steps and the record's revision. A record that still carries its steps inline (one
|
||||
// written before H-A-187) is read whole, said once in the journal, and split at its next write.
|
||||
func (s *MoverStore) Get(key string) (seat.Pipeline, uint64, error) {
|
||||
defer s.lock(key)()
|
||||
raw, rev, err := s.Raw.Get(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
if raw == nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return seat.Pipeline{}, 0, fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, 0, err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: revs}
|
||||
if len(p.Steps) > 0 {
|
||||
if !s.inline[key] {
|
||||
s.inline[key] = true
|
||||
s.logf("the pipeline %s carries its %d steps inline in its record: they are read whole and split into one key per mover at its next write (H-A-187)", key, len(p.Steps))
|
||||
}
|
||||
return p, rev, nil
|
||||
}
|
||||
if p.Steps == nil {
|
||||
p.Steps = []seat.StepResult{}
|
||||
}
|
||||
return p, rev, composeSteps(&p, key, vals)
|
||||
}
|
||||
|
||||
// Update writes p as pipeline key's record and its mover keys, each by compare-and-set: nothing is written, ErrChanged,
|
||||
// when the record or a mover key was written since the read revision gave. A record carrying its steps inline has its
|
||||
// mover keys written before the record, so its steps are never lost between the two.
|
||||
func (s *MoverStore) Update(key string, p seat.Pipeline, revision uint64) (uint64, error) {
|
||||
defer s.lock(key)()
|
||||
record, want, err := split(key, p)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, cur, err := s.Raw.Get(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if raw == nil {
|
||||
return 0, fmt.Errorf("%w: %s", ErrNoPipeline, key)
|
||||
}
|
||||
if cur != revision {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// A mover key written since the read is another write between: the read named its revision.
|
||||
if seen, ok := s.read[key]; ok && seen.rev == revision {
|
||||
for k, r := range revs {
|
||||
if seen.movers[k] != r {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
}
|
||||
for k := range seen.movers {
|
||||
if _, ok := revs[k]; !ok {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
}
|
||||
}
|
||||
var was seat.Pipeline
|
||||
inline := json.Unmarshal(raw, &was) == nil && len(was.Steps) > 0
|
||||
var after map[string]uint64
|
||||
if inline {
|
||||
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
rev, err := s.Raw.Update(key, record, revision)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !inline {
|
||||
if after, err = s.writeMovers(key, want, vals, revs); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
s.logf("the pipeline %s's steps are split into %d mover keys and its record no longer carries them (H-A-187)", key, len(want))
|
||||
delete(s.inline, key)
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
return rev, nil
|
||||
}
|
||||
|
||||
// writeMovers writes each mover key whose steps changed and deletes each that p no longer names, by compare-and-set on the
|
||||
// revisions read, and answers every mover key's revision after it.
|
||||
func (s *MoverStore) writeMovers(key string, want, vals map[string][]byte, revs map[string]uint64) (map[string]uint64, error) {
|
||||
after := map[string]uint64{}
|
||||
for _, k := range slices.Sorted(maps.Keys(want)) {
|
||||
v := want[k]
|
||||
old, ok := vals[k]
|
||||
var rev uint64
|
||||
var err error
|
||||
switch {
|
||||
case ok && string(old) == string(v):
|
||||
after[k] = revs[k]
|
||||
continue
|
||||
case ok:
|
||||
rev, err = s.Raw.Update(k, v, revs[k])
|
||||
default:
|
||||
rev, err = s.Raw.Create(k, v)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s are not written: %w", k, err)
|
||||
}
|
||||
after[k] = rev
|
||||
}
|
||||
for k := range vals {
|
||||
if _, ok := want[k]; !ok {
|
||||
if err := s.Raw.Delete(k); err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s are not deleted: %w", k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return after, nil
|
||||
}
|
||||
|
||||
// Create writes a new pipeline, its record by compare-and-set on its absence (ErrExists: the second start writes nothing,
|
||||
// §54), then its mover keys; a mover key an earlier pipeline of the key left is written over, said in the journal.
|
||||
func (s *MoverStore) Create(key string, value []byte) error {
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(value, &p); err != nil {
|
||||
return fmt.Errorf("the pipeline %s is no pipeline: %w", key, err)
|
||||
}
|
||||
defer s.lock(key)()
|
||||
record, want, err := split(key, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := s.Raw.Create(key, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vals, revs, err := s.moverKeys(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k := range vals {
|
||||
s.logf("%s was left by an earlier pipeline of the key: it is written over (H-A-187)", k)
|
||||
}
|
||||
after, err := s.writeMovers(key, want, vals, revs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.read[key] = readAt{rev: rev, movers: after}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes pipeline key's mover keys, then its record (§330): a delete stopped between leaves the record, which the
|
||||
// next ageing deletes again.
|
||||
func (s *MoverStore) Delete(key string) error {
|
||||
unlock := s.lock(key)
|
||||
vals, _, err := s.moverKeys(key)
|
||||
if err == nil {
|
||||
for k := range vals {
|
||||
if err = s.Raw.Delete(k); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = s.Raw.Delete(key)
|
||||
}
|
||||
delete(s.read, key)
|
||||
delete(s.inline, key)
|
||||
unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
// Keys are the keys of the state pipelines matching filter, `*` one key part.
|
||||
func (s *MoverStore) Keys(filter string) ([]string, error) { return s.Raw.Keys(filter) }
|
||||
@@ -0,0 +1,254 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// memRaw is the state pipelines as bytes, one revision sequence for the bucket as a key-value's stream has.
|
||||
type memRaw struct {
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func newMemRaw() *memRaw { return &memRaw{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memRaw) Get(key string) ([]byte, uint64, error) {
|
||||
v, ok := m.vals[key]
|
||||
if !ok {
|
||||
return nil, 0, nil
|
||||
}
|
||||
return append([]byte(nil), v...), m.revs[key], nil
|
||||
}
|
||||
|
||||
func (m *memRaw) put(key string, value []byte) uint64 {
|
||||
m.seq++
|
||||
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.seq
|
||||
return m.seq
|
||||
}
|
||||
|
||||
func (m *memRaw) Create(key string, value []byte) (uint64, error) {
|
||||
if _, ok := m.vals[key]; ok {
|
||||
return 0, ErrExists
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Update(key string, value []byte, rev uint64) (uint64, error) {
|
||||
if m.revs[key] != rev {
|
||||
return 0, ErrChanged
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Delete(key string) error {
|
||||
delete(m.vals, key)
|
||||
delete(m.revs, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memRaw) Keys(filter string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range m.vals {
|
||||
if ok, _ := matchKey(filter, k); ok {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordSteps is the steps the raw record of key carries itself.
|
||||
func recordSteps(t *testing.T, m *memRaw, key string) int {
|
||||
t.Helper()
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(m.vals[key], &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(p.Steps)
|
||||
}
|
||||
|
||||
// TestMoverKeys holds the state pipelines' shape to H-A-187: a pipeline's record keeps its movers and outcome, each mover's
|
||||
// steps their own key written by compare-and-set as the record is, a read composing them, a record carrying its steps inline
|
||||
// read whole and split at its next write, one row per case.
|
||||
func TestMoverKeys(t *testing.T) {
|
||||
const key = "main.7"
|
||||
two := func() seat.Pipeline {
|
||||
return seat.Pipeline{V: 1, Pipeline: key, Branch: "main", Source: 7, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "beta"}, {Module: "alpha"}},
|
||||
Steps: []seat.StepResult{{Module: "alpha", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
|
||||
{Module: "beta", Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeRunning},
|
||||
{Module: "beta", Step: seat.StepResultStepApproval, Outcome: seat.StepResultOutcomeNotAsked}}}
|
||||
}
|
||||
created := func(t *testing.T) (*memRaw, *MoverStore, *[]string) {
|
||||
m, logs := newMemRaw(), &[]string{}
|
||||
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
|
||||
raw, _ := json.Marshal(two())
|
||||
if err := s.Create(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, s, logs
|
||||
}
|
||||
steps := func(p seat.Pipeline) string {
|
||||
var out []string
|
||||
for _, st := range p.Steps {
|
||||
out = append(out, st.Module+" "+st.Step+" "+st.Outcome)
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
rows := []struct {
|
||||
name string
|
||||
run func(t *testing.T) string
|
||||
want string
|
||||
}{
|
||||
{"a write keeps the record's steps empty and each mover's under its own key; a read composes them in the movers' order", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, _, err := s.Get(key)
|
||||
return fmt.Sprintf("record %d, alpha %t, beta %t: %s %v", recordSteps(t, m, key), m.vals["steps.main.7.alpha"] != nil,
|
||||
m.vals["steps.main.7.beta"] != nil, steps(p), err)
|
||||
}, "record 0, alpha true, beta true: beta build running; beta approval not asked; alpha build running <nil>"},
|
||||
{"an update writes the mover whose steps changed and leaves the other's key as it was", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
beta := m.revs["steps.main.7.beta"]
|
||||
p.Steps[2].Outcome = seat.StepResultOutcomePassed
|
||||
_, err := s.Update(key, p, rev)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("beta kept %t: %s %v", m.revs["steps.main.7.beta"] == beta, steps(q), err)
|
||||
}, "beta kept true: beta build running; beta approval not asked; alpha build passed <nil>"},
|
||||
{"a mover key written since the read refuses the update, nothing written", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
m.put("steps.main.7.alpha", m.vals["steps.main.7.alpha"])
|
||||
p.Outcome = seat.PipelineOutcomePassed
|
||||
_, err := s.Update(key, p, rev)
|
||||
return fmt.Sprintf("%t, record at %t", errors.Is(err, ErrChanged), m.revs[key] == rev)
|
||||
}, "true, record at true"},
|
||||
{"a record written since the read refuses the update", func(t *testing.T) string {
|
||||
_, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
_, err := s.Update(key, p, rev)
|
||||
return fmt.Sprint(errors.Is(err, ErrChanged))
|
||||
}, "true"},
|
||||
{"a record carrying its steps inline is read whole, said once, and split at its next write", func(t *testing.T) string {
|
||||
m, logs := newMemRaw(), &[]string{}
|
||||
s := &MoverStore{Raw: m, Logf: func(f string, a ...any) { *logs = append(*logs, fmt.Sprintf(f, a...)) }}
|
||||
raw, _ := json.Marshal(two())
|
||||
m.put(key, raw)
|
||||
p, rev, _ := s.Get(key)
|
||||
_, _, _ = s.Get(key)
|
||||
read := fmt.Sprintf("read %d steps, %d lines", len(p.Steps), len(*logs))
|
||||
_, err := s.Update(key, p, rev)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("%s; written: record %d, %s, split said %t %v", read, recordSteps(t, m, key), steps(q),
|
||||
strings.Contains(strings.Join(*logs, "\n"), "split into 2 mover keys"), err)
|
||||
}, "read 3 steps, 1 lines; written: record 0, beta build running; beta approval not asked; alpha build running, split said true <nil>"},
|
||||
{"a second create writes nothing", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p := two()
|
||||
p.Steps[0].Outcome = seat.StepResultOutcomeFailed
|
||||
raw, _ := json.Marshal(p)
|
||||
err := s.Create(key, raw)
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("%t %d: %s", errors.Is(err, ErrExists), len(m.vals), steps(q))
|
||||
}, "true 3: beta build running; beta approval not asked; alpha build running"},
|
||||
{"a delete removes the mover keys and the record", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
err := s.Delete(key)
|
||||
return fmt.Sprintf("%d %v", len(m.vals), err)
|
||||
}, "0 <nil>"},
|
||||
{"a record of 129 movers never grows with their steps (main.26)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
p.Movers, p.Steps = nil, nil
|
||||
for i := range 129 {
|
||||
mod := fmt.Sprintf("module-%03d", i)
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
|
||||
for _, step := range []string{seat.StepResultStepBuild, seat.StepResultStepLint, seat.StepResultStepUnit,
|
||||
seat.StepResultStepIntegration, seat.StepResultStepValidate, seat.StepResultStepComposition,
|
||||
seat.StepResultStepReview, seat.StepResultStepApproval} {
|
||||
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: step, Artifact: mod + ":program",
|
||||
Outcome: seat.StepResultOutcomeFailed, Why: strings.Repeat("why ", 250)})
|
||||
}
|
||||
}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
largest := 0
|
||||
for k, v := range m.vals {
|
||||
if IsStepsKey(k) && len(v) > largest {
|
||||
largest = len(v)
|
||||
}
|
||||
}
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("record under 64 KiB %t, a mover key under 16 KiB %t, %d steps read", len(m.vals[key]) < 64<<10,
|
||||
largest < 16<<10, len(q.Steps))
|
||||
}, "record under 64 KiB true, a mover key under 16 KiB true, 1032 steps read"},
|
||||
{"a mover's unlogged entries live under its mover key, each filed by the mover it names (H-A-189, H-A-190)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
|
||||
// Two movers' approvals end alike at one moment; each entry names its mover.
|
||||
p.Unlogged = []seat.LogEntry{{Pipeline: key, Module: "beta", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at},
|
||||
{Pipeline: key, Module: "alpha", Step: seat.StepResultStepApproval, What: seat.LogEntryWhatEnded, At: at.Add(time.Second)}}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
var rec seat.Pipeline
|
||||
_ = json.Unmarshal(m.vals[key], &rec)
|
||||
beta, _ := seat.DecodePipelinesByStepsPipelineMoverValue(m.vals["steps.main.7.beta"])
|
||||
q, _, _ := s.Get(key)
|
||||
var got []string
|
||||
for _, e := range q.Unlogged {
|
||||
got = append(got, e.Module)
|
||||
}
|
||||
return fmt.Sprintf("record %d unlogged, beta's key %d (%s), read %s", len(rec.Unlogged), len(beta.Unlogged),
|
||||
beta.Unlogged[0].Module, strings.Join(got, " "))
|
||||
}, "record 0 unlogged, beta's key 1 (beta), read beta alpha"},
|
||||
{"a 129-mover pipeline with 1,000 unlogged entries keeps its record small (H-A-189)", func(t *testing.T) string {
|
||||
m, s, _ := created(t)
|
||||
p, rev, _ := s.Get(key)
|
||||
p.Movers, p.Steps, p.Unlogged = nil, nil, nil
|
||||
at := time.Date(2026, 11, 20, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 129 {
|
||||
mod := fmt.Sprintf("module-%03d", i)
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: mod, Commit: strings.Repeat("c", 40), Fingerprint: "fp-" + mod})
|
||||
p.Steps = append(p.Steps, seat.StepResult{Module: mod, Step: seat.StepResultStepBuild, Outcome: seat.StepResultOutcomeNotAsked})
|
||||
}
|
||||
for i := range 1000 {
|
||||
mod := fmt.Sprintf("module-%03d", i%129)
|
||||
p.Unlogged = append(p.Unlogged, seat.LogEntry{Pipeline: key, Module: mod, Step: seat.StepResultStepBuild, Artifact: mod + ":program",
|
||||
What: seat.LogEntryWhatEnded, Outcome: seat.StepResultOutcomeNotAsked, Output: strings.Repeat("o", 200), At: at.Add(time.Duration(i) * time.Millisecond)})
|
||||
}
|
||||
if _, err := s.Update(key, p, rev); err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
q, _, _ := s.Get(key)
|
||||
return fmt.Sprintf("record under 64 KiB %t, %d unlogged read", len(m.vals[key]) < 64<<10, len(q.Unlogged))
|
||||
}, "record under 64 KiB true, 1000 unlogged read"},
|
||||
{"mover keys are told from records of every key form", func(t *testing.T) string {
|
||||
var out []string
|
||||
for _, k := range []string{"steps.main.7.alpha", "steps.main.run.ab12.alpha", "steps.main.7.published.p.v.alpha",
|
||||
"main.7", "steps.7", "steps.run.ab12", "steps.7.published.p.v"} {
|
||||
out = append(out, fmt.Sprint(IsStepsKey(k)))
|
||||
}
|
||||
return strings.Join(out, " ")
|
||||
}, "true true true false false false false"},
|
||||
}
|
||||
for _, r := range rows {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
if got := r.run(t); got != r.want {
|
||||
t.Errorf("got %s\nwant %s", got, r.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,8 @@ func (r Reads) fills() ([]seat.Fill, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83).
|
||||
// Show is one pipeline whole, with its steps and the names of their outputs in step-outputs (§83), or, naming a mover, the
|
||||
// pipeline's record with that mover's steps alone (H-A-187).
|
||||
func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
p, ok, err := r.pipeline(in.Pipeline)
|
||||
if err != nil {
|
||||
@@ -52,6 +53,13 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
if err != nil {
|
||||
return seat.ShowAnswer{}, err
|
||||
}
|
||||
// Named, a mover is answered by its steps alone (H-A-187): one mover's of a pipeline of a hundred movers.
|
||||
if m := strings.TrimSpace(in.Mover); m != "" {
|
||||
if !Moves(p, m) {
|
||||
return seat.ShowAnswer{}, fmt.Errorf("the pipeline %s does not move %s", in.Pipeline, m)
|
||||
}
|
||||
p.Steps = stepsOf(p, m)
|
||||
}
|
||||
a := seat.ShowAnswer{Pipeline: p, Fill: fills}
|
||||
for _, s := range p.Steps {
|
||||
if s.Output != "" {
|
||||
@@ -62,7 +70,7 @@ func (r Reads) Show(in seat.ShowInput) (seat.ShowAnswer, error) {
|
||||
}
|
||||
|
||||
// List is the pipelines and runs of a module, a branch, a commit or a run, newest started first, at most Limit when it
|
||||
// names one (§83).
|
||||
// names one (§83): each one's record with its movers and steps counted, its steps empty (H-A-191).
|
||||
func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
|
||||
all, err := r.pipelines()
|
||||
if err != nil {
|
||||
@@ -72,6 +80,9 @@ func (r Reads) List(in seat.ListInput) (seat.ListAnswer, error) {
|
||||
for _, p := range all {
|
||||
if (in.Module == "" || Moves(p, in.Module)) && (in.Branch == "" || p.Branch == in.Branch) &&
|
||||
(in.Commit == "" || movesCommit(p, in.Commit)) && (in.Run == "" || p.Run == in.Run) {
|
||||
// Each pipeline's record with its movers and steps counted, never its steps: show gives those (H-A-191).
|
||||
p.MoverCount, p.StepCount = int64(len(p.Movers)), int64(len(p.Steps))
|
||||
p.Steps, p.Unlogged = []seat.StepResult{}, nil
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
@@ -178,6 +189,16 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
|
||||
if key == "" {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("name the pipeline's key, <branch>.<revision>.<module>")
|
||||
}
|
||||
all, err := r.Pipelines.Keys()
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s cannot be read: %w", seat.StatePipelines, err)
|
||||
}
|
||||
return r.pipelineOf(key, all)
|
||||
}
|
||||
|
||||
// pipelineOf is pipeline key, its steps composed from the mover keys among all that it names (H-A-187); a record that
|
||||
// still carries its steps inline is read whole.
|
||||
func (r Reads) pipelineOf(key string, all []string) (seat.Pipeline, bool, error) {
|
||||
raw, err := r.Pipelines.Value(key)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, key, err)
|
||||
@@ -192,7 +213,24 @@ func (r Reads) pipeline(key string) (seat.Pipeline, bool, error) {
|
||||
if p.Pipeline == "" {
|
||||
p.Pipeline = key
|
||||
}
|
||||
return p, true, nil
|
||||
if len(p.Steps) > 0 {
|
||||
return p, true, nil
|
||||
}
|
||||
prefix := stepsPrefix(key)
|
||||
vals := map[string][]byte{}
|
||||
for _, k := range all {
|
||||
if !strings.HasPrefix(k, prefix) || strings.Contains(k[len(prefix):], ".") {
|
||||
continue
|
||||
}
|
||||
v, err := r.Pipelines.Value(k)
|
||||
if err != nil {
|
||||
return seat.Pipeline{}, false, fmt.Errorf("%s %s cannot be read: %w", seat.StatePipelines, k, err)
|
||||
}
|
||||
if v != nil {
|
||||
vals[k] = v
|
||||
}
|
||||
}
|
||||
return p, true, composeSteps(&p, key, vals)
|
||||
}
|
||||
|
||||
func (r Reads) pipelines() ([]seat.Pipeline, error) {
|
||||
@@ -203,7 +241,10 @@ func (r Reads) pipelines() ([]seat.Pipeline, error) {
|
||||
sort.Strings(keys)
|
||||
var out []seat.Pipeline
|
||||
for _, k := range keys {
|
||||
p, ok, err := r.pipeline(k)
|
||||
if IsStepsKey(k) {
|
||||
continue
|
||||
}
|
||||
p, ok, err := r.pipelineOf(k, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -49,6 +50,15 @@ func TestReads(t *testing.T) {
|
||||
Outcome: seat.PipelineOutcomePassed, Started: &t2, Movers: []seat.Mover{{Module: "alpha", Commit: "b3"}}})
|
||||
put(t, pipes, "main.7.gamma", seat.Pipeline{Pipeline: "main.7.gamma", Branch: "main",
|
||||
Outcome: seat.PipelineOutcomeRunning, Started: &t1, Movers: []seat.Mover{{Module: "gamma", Commit: "c7"}}})
|
||||
// A pipeline kept as H-A-187 keeps it: its record's steps empty, each mover's under steps.<pipeline>.<mover>.
|
||||
put(t, pipes, "next.8", seat.Pipeline{Pipeline: "next.8", Branch: "next", Source: 8, Outcome: seat.PipelineOutcomeRunning,
|
||||
Started: &t0, Steps: []seat.StepResult{}, Movers: []seat.Mover{{Module: "delta", Commit: "n8"}, {Module: "epsilon", Commit: "n8"}}})
|
||||
put(t, pipes, "steps.next.8.epsilon", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "epsilon",
|
||||
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed, Output: "next.8.build.epsilon:program"}}})
|
||||
put(t, pipes, "steps.next.8.delta", seat.PipelinesByStepsPipelineMoverValue{Steps: []seat.StepResult{{Module: "delta",
|
||||
Step: seat.StepResultStepBuild, Status: "required", Outcome: seat.StepResultOutcomePassed}, {Module: "delta",
|
||||
Step: seat.StepResultStepApproval, Status: "required", Outcome: seat.StepResultOutcomeWaiting, On: seat.WaitOnPerson, Started: &t2}},
|
||||
Unlogged: []seat.LogEntry{{Pipeline: "next.8", Module: "delta", Step: seat.StepResultStepBuild, What: seat.LogEntryWhatEnded, At: t1}}})
|
||||
put(t, kept, "unit.sha256-aa", seat.KeptResult{Level: "unit", Digest: "sha256-aa", Package: "alpha/x"})
|
||||
put(t, kept, "integration.sha256-bb", seat.KeptResult{Level: "integration", Digest: "sha256-bb", Package: "alpha/y"})
|
||||
r := Reads{Pipelines: pipes, Kept: kept}
|
||||
@@ -73,6 +83,35 @@ func TestReads(t *testing.T) {
|
||||
_, err := r.Show(seat.ShowInput{Pipeline: "main.9.alpha"})
|
||||
return "", err
|
||||
}, `error: no pipeline "main.9.alpha" in pipelines`},
|
||||
{"show composes a pipeline's steps from its mover keys, in its movers' order (H-A-187)", func() (string, error) {
|
||||
a, err := r.Show(seat.ShowInput{Pipeline: "next.8"})
|
||||
var s []string
|
||||
for _, st := range a.Pipeline.Steps {
|
||||
s = append(s, st.Module+" "+st.Step)
|
||||
}
|
||||
return strings.Join(s, ", ") + " | " + strings.Join(a.Outputs, ","), err
|
||||
}, "delta build, delta approval, epsilon build | next.8.build.epsilon:program"},
|
||||
{"show naming a mover answers its steps alone", func() (string, error) {
|
||||
a, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "epsilon"})
|
||||
var s []string
|
||||
for _, st := range a.Pipeline.Steps {
|
||||
s = append(s, st.Module+" "+st.Step)
|
||||
}
|
||||
return fmt.Sprintf("%s, %d movers", strings.Join(s, ", "), len(a.Pipeline.Movers)), err
|
||||
}, "epsilon build, 2 movers"},
|
||||
{"show refuses a mover the pipeline does not move", func() (string, error) {
|
||||
_, err := r.Show(seat.ShowInput{Pipeline: "next.8", Mover: "alpha"})
|
||||
return "", err
|
||||
}, "error: the pipeline next.8 does not move alpha"},
|
||||
{"list names a pipeline, never a mover key, its record with its movers and steps counted and no step (H-A-191)", func() (string, error) {
|
||||
a, err := r.List(seat.ListInput{Branch: "next"})
|
||||
var s []string
|
||||
for _, p := range a.Pipelines {
|
||||
s = append(s, fmt.Sprintf("%s %d movers %d steps, %d steps and %d unlogged answered", p.Pipeline, p.MoverCount, p.StepCount,
|
||||
len(p.Steps), len(p.Unlogged)))
|
||||
}
|
||||
return strings.Join(s, "; "), err
|
||||
}, "next.8 2 movers 3 steps, 0 steps and 0 unlogged answered"},
|
||||
{"list by module, newest first", func() (string, error) {
|
||||
a, err := r.List(seat.ListInput{Module: "alpha"})
|
||||
return names(a.Pipelines), err
|
||||
@@ -92,7 +131,7 @@ func TestReads(t *testing.T) {
|
||||
s = append(s, w.Pipeline+" "+w.Step+" "+w.On+" "+w.Since.Format("15:04"))
|
||||
}
|
||||
return strings.Join(s, ";"), err
|
||||
}, "main.7.alpha approval person 10:01"},
|
||||
}, "main.7.alpha approval person 10:01;next.8 approval person 10:02"},
|
||||
{"why-carried names where a carried step came from", func() (string, error) {
|
||||
a, err := r.WhyCarried(seat.WhyCarriedInput{Pipeline: "main.7.alpha", Step: "unit", Artifact: "alpha:program"})
|
||||
return a.CarriedFrom, err
|
||||
|
||||
@@ -441,7 +441,7 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
||||
|
||||
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
|
||||
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
|
||||
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
||||
return seat.LogEntry{Pipeline: key, Module: s.Module, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
||||
Output: s.Output, Kept: kept, At: at}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,9 +7,10 @@ import (
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// sameEntry says whether two step-log entries are one: the same pipeline, step, artifact, end or start and time.
|
||||
// sameEntry says whether two step-log entries are one: the same pipeline, mover (H-A-190), step, artifact, end or start and
|
||||
// time.
|
||||
func sameEntry(a, b seat.LogEntry) bool {
|
||||
return a.Pipeline == b.Pipeline && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
|
||||
return a.Pipeline == b.Pipeline && a.Module == b.Module && a.Step == b.Step && a.Artifact == b.Artifact && a.What == b.What && a.At.Equal(b.At)
|
||||
}
|
||||
|
||||
// appendUnlogged appends the unlogged entries of pipeline key to the step log in their order and clears each appended one in a
|
||||
|
||||
Reference in New Issue
Block a user