Files
mesh-delivery/modules/mesh-delivery-view/internal/view/pipelines.go
T
jschoubben 7a32a8b783 view: keep the pipeline runner's pipelines and serve view_pipelines
The view hears the seat mesh-pipeline-runner's pipeline-started, step-started, step-ended and pipeline-ended and the build seats' step-progress, keeps each pipeline whole in its own state pipelines (newest 500) and the latest progress per running ask in pipeline-progress (200), and view_pipelines reads them.

Builds: the lead's handover of 2026-11-16 (part 1 of the delivery view through its own pipeline); design 47 G, Pipelines per module (no milestone line cited by the handover)
2026-11-16 21:37:07 +01:00

475 lines
16 KiB
Go

package view
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The view keeps the pipeline runner's pipelines in its own state `pipelines` and the latest progress of each running ask in its
// state `pipeline-progress`, from the seat mesh-pipeline-runner's events pipeline-started, step-started, step-ended and
// pipeline-ended and the build seats' step-progress; view_pipelines reads them (design 47 G *Pipelines per module*).
// PipelinesReadTool reads the view's pipelines, each whole with its steps and the progress of the steps that run.
const PipelinesReadTool = "view_pipelines"
// MaxPipelines is how many pipelines the state `pipelines` keeps: when more are kept, the longest unheard ones go. MaxProgress is the
// same for the state `pipeline-progress`, whose entry also goes when its step ends. DefaultPipelinesLimit is how many a read gives
// when no limit is asked, MaxPipelinesLimit the most.
const (
MaxPipelines = 500
MaxProgress = 200
DefaultPipelinesLimit = 50
MaxPipelinesLimit = 500
pruneSlack = 50 // a state over its cap by less than this is not cut yet, so a replay does not read it at every new key
pipelineWrites = 5 // how often a write over a changed key is tried again before the event is left for the bus to give again
)
// The seat events the view hears for its pipelines, and the build seats whose step-progress it hears.
const (
EventStepProgress = "step-progress"
)
// PipelineSubjects are the subjects of the pipeline runner's events the view hears, each the seat contract's.
func PipelineSubjects() []string {
return []string{meshpipelinerunner.SubjectPipelineStartedAll, meshpipelinerunner.SubjectStepStartedAll,
meshpipelinerunner.SubjectStepEndedAll, meshpipelinerunner.SubjectPipelineEndedAll}
}
// ProgressSubjects are the build seats' step-progress subjects the view hears, each the seat contract's.
func ProgressSubjects() []string {
return []string{nodebuildgo.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll}
}
// PipelineRecord is the value of a pipelines key: the pipeline as the runner said it (the steps with their outcomes, why, times)
// and Heard, when the view last kept an event of it.
type PipelineRecord struct {
meshpipelinerunner.Pipeline
Heard time.Time `json:"heard"`
}
// ProgressRecord is the value of a pipeline-progress key: the latest progress said of one ask, with the seat and the node that said it.
type ProgressRecord struct {
Ask string `json:"ask"`
Asker string `json:"asker,omitempty"`
Phase string `json:"phase"`
At time.Time `json:"at"`
Seat string `json:"seat"`
Node string `json:"node,omitempty"`
}
// ErrNotKept is what a store's Get says of a key it does not hold.
var ErrNotKept = errors.New("not kept")
// PipelineStore is the view's own bucket as the pipelines are kept in it: a write over a revision is refused when the key moved
// (revision 0 creates it).
type PipelineStore interface {
Get(key string) (value []byte, revision uint64, err error)
Put(key string, value []byte, revision uint64) error
Delete(key string) error
Keys() ([]string, error)
}
// ProgressKey is the key of an ask's progress.
func ProgressKey(ask string) string { return keys.EscapeName(ask) }
// ApplyPipelineEvent keeps one of the pipeline runner's events (its subject names which) in pipelines, and the progress of an ask a
// step's end or the pipeline's end closes in progress. An event that cannot be kept is an error and the bus gives it again.
func ApplyPipelineEvent(subject string, body []byte, pipelines, progress PipelineStore, now time.Time) error {
parts := strings.Split(subject, ".")
if len(parts) < 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
return fmt.Errorf("%s is no seat event's subject", subject)
}
event := parts[4]
switch event {
case meshpipelinerunner.EventPipelineStarted:
var e meshpipelinerunner.PipelineStarted
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
return changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
if e.Published != nil {
p.Published = e.Published
}
if p.Started == nil {
p.Started = e.At
}
for _, s := range e.Steps {
if stepIndex(p.Steps, s.Step, s.Artifact) < 0 {
p.Steps = append(p.Steps, meshpipelinerunner.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status})
}
}
})
case meshpipelinerunner.EventStepStarted, meshpipelinerunner.EventStepEnded:
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
i := stepIndex(p.Steps, e.Result.Step, e.Result.Artifact)
switch {
case i < 0:
p.Steps = append(p.Steps, e.Result)
case p.Outcome != meshpipelinerunner.PipelineEndedOutcomeRunning && p.Outcome != "" && p.Steps[i].Outcome != "":
// The pipeline's end said every step whole: a step's event heard after it changes nothing.
default:
p.Steps[i] = e.Result
}
})
if err != nil {
return err
}
if event == meshpipelinerunner.EventStepEnded && e.Result.Ask != "" {
return deleteKey(progress, ProgressKey(e.Result.Ask))
}
return nil
case meshpipelinerunner.EventPipelineEnded:
var e meshpipelinerunner.PipelineEnded
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
p.Outcome, p.Why, p.Steps, p.Tests, p.OverBudget, p.Widens = e.Outcome, e.Why, e.Steps, e.Tests, e.OverBudget, e.Widens
p.EndCount, p.Ended = e.EndCount, e.Ended
if e.Started != nil {
p.Started = e.Started
}
})
if err != nil {
return err
}
for _, s := range e.Steps {
if s.Ask != "" {
if err := deleteKey(progress, ProgressKey(s.Ask)); err != nil {
return err
}
}
}
return nil
}
return fmt.Errorf("%s is no pipeline event the view hears", subject)
}
// fillPipeline puts what an event names of its pipeline into p; a pipeline first heard of by a step's event has an outcome of
// running until its end says another.
func fillPipeline(p *PipelineRecord, pipeline, module, branch, commit, group, run, fingerprint string, revision, source int64) {
q := &p.Pipeline
q.Pipeline, q.Module, q.Branch, q.Commit = pipeline, module, branch, commit
q.Group, q.Run, q.Fingerprint, q.Revision, q.Source = group, run, fingerprint, revision, source
if q.Outcome == "" {
q.Outcome = meshpipelinerunner.PipelineEndedOutcomeRunning
}
}
func stepIndex(steps []meshpipelinerunner.StepResult, step, artifact string) int {
for i, s := range steps {
if s.Step == step && s.Artifact == artifact {
return i
}
}
return -1
}
// changePipeline reads a pipeline's record (a new one when none), lets change alter it and writes it over the revision it read,
// again when the key moved in between; a new key over the cap cuts the oldest heard (prune).
func changePipeline(store PipelineStore, key string, now time.Time, change func(*PipelineRecord)) error {
if strings.TrimSpace(key) == "" {
return errors.New("the event names no pipeline")
}
var err error
for try := 0; try < pipelineWrites; try++ {
var rec PipelineRecord
raw, rev, gerr := store.Get(key)
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
return gerr
}
if rev != 0 && len(raw) > 0 {
if jerr := json.Unmarshal(raw, &rec); jerr != nil {
return fmt.Errorf("the pipeline %s is kept in a form that does not read: %w", key, jerr)
}
}
change(&rec)
rec.Heard = now.UTC()
out, jerr := json.Marshal(rec)
if jerr != nil {
return jerr
}
if err = store.Put(key, out, rev); err == nil {
if rev == 0 {
return prunePipelines(store, MaxPipelines)
}
return nil
}
}
return fmt.Errorf("the pipeline %s moved while it was kept: %w", key, err)
}
func deleteKey(store PipelineStore, key string) error {
if err := store.Delete(key); err != nil && !errors.Is(err, ErrNotKept) {
return err
}
return nil
}
// prunePipelines cuts the pipelines to max when more than max plus the slack are kept: the ones heard longest ago go.
func prunePipelines(store PipelineStore, max int) error {
all, err := store.Keys()
if err != nil || len(all) <= max+pruneSlack {
return err
}
type kept struct {
key string
heard time.Time
}
var ks []kept
for _, k := range all {
raw, _, err := store.Get(k)
if err != nil {
if errors.Is(err, ErrNotKept) {
continue
}
return err
}
var h struct {
Heard time.Time `json:"heard"`
}
_ = json.Unmarshal(raw, &h)
ks = append(ks, kept{k, h.Heard})
}
sort.Slice(ks, func(i, j int) bool { return ks[i].heard.Before(ks[j].heard) })
for i := 0; i < len(ks)-max; i++ {
if err := deleteKey(store, ks[i].key); err != nil {
return err
}
}
return nil
}
// ApplyProgress keeps one build seat's step-progress (a subject mesh.seat.<seat>.event.step-progress.<node>): the latest per ask, so a
// progress older than the one kept changes nothing.
func ApplyProgress(subject string, body []byte, progress PipelineStore) error {
parts := strings.Split(subject, ".")
if len(parts) != 6 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" || parts[4] != EventStepProgress {
return fmt.Errorf("%s is no step-progress subject", subject)
}
var e struct {
Ask string `json:"ask"`
Asker string `json:"asker"`
Phase string `json:"phase"`
At time.Time `json:"at"`
}
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
if e.Ask == "" || e.Phase == "" {
return fmt.Errorf("%s names no ask or no phase", subject)
}
key := ProgressKey(e.Ask)
rec := ProgressRecord{Ask: e.Ask, Asker: e.Asker, Phase: e.Phase, At: e.At.UTC(), Seat: parts[2], Node: parts[5]}
out, err := json.Marshal(rec)
if err != nil {
return err
}
for try := 0; try < pipelineWrites; try++ {
raw, rev, gerr := progress.Get(key)
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
return gerr
}
if rev != 0 {
var have ProgressRecord
if json.Unmarshal(raw, &have) == nil && !have.At.Before(rec.At) {
return nil
}
}
if err = progress.Put(key, out, rev); err == nil {
if rev == 0 {
return pruneProgress(progress, MaxProgress)
}
return nil
}
}
return fmt.Errorf("the progress of %s moved while it was kept: %w", e.Ask, err)
}
// pruneProgress cuts the progress to max when more than max plus the slack are kept: the oldest said go (an ask whose end was never
// heard).
func pruneProgress(store PipelineStore, max int) error {
all, err := store.Keys()
if err != nil || len(all) <= max+pruneSlack {
return err
}
type kept struct {
key string
at time.Time
}
var ks []kept
for _, k := range all {
raw, _, err := store.Get(k)
if err != nil {
if errors.Is(err, ErrNotKept) {
continue
}
return err
}
var p ProgressRecord
_ = json.Unmarshal(raw, &p)
ks = append(ks, kept{k, p.At})
}
sort.Slice(ks, func(i, j int) bool { return ks[i].at.Before(ks[j].at) })
for i := 0; i < len(ks)-max; i++ {
if err := deleteKey(store, ks[i].key); err != nil {
return err
}
}
return nil
}
// PipelinesQuery is what view_pipelines asks: each field narrows, none means all.
type PipelinesQuery struct {
Branch, Module, Outcome string
Since time.Time // pipelines the view kept an event of at or after it
Limit int
}
// StepProgressView is a running step's latest progress: its phase and when it was said.
type StepProgressView struct {
Phase string `json:"phase"`
At time.Time `json:"at"`
Seat string `json:"seat"`
Node string `json:"node,omitempty"`
}
// PipelineStepView is a step as the runner said it, with the progress of the step's ask while it runs.
type PipelineStepView struct {
meshpipelinerunner.StepResult
Progress *StepProgressView `json:"progress,omitempty"`
}
// PipelineView is a pipeline whole with its steps, Heard when the view last kept an event of it.
type PipelineView struct {
meshpipelinerunner.Pipeline
Steps []PipelineStepView `json:"steps"`
Heard time.Time `json:"heard"`
}
// PipelinesAnswer is view_pipelines' answer: the pipelines newest first, Matching how many fit before the limit cut them.
type PipelinesAnswer struct {
Pipelines []PipelineView `json:"pipelines"`
Matching int `json:"matching"`
Cut bool `json:"cut,omitempty"`
// Unreadable names the keys of a state that do not read.
Unreadable []string `json:"unreadable,omitempty"`
}
// ParsePipelinesQuery reads the tool's arguments; since is an RFC 3339 time, limit a whole number.
func ParsePipelinesQuery(args map[string]any) (PipelinesQuery, error) {
var q PipelinesQuery
q.Branch, _ = args["branch"].(string)
q.Module, _ = args["module"].(string)
q.Outcome, _ = args["outcome"].(string)
if raw, _ := args["since"].(string); strings.TrimSpace(raw) != "" {
at, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw))
if err != nil {
return q, fmt.Errorf("since %q is not an RFC 3339 time: %w", raw, err)
}
q.Since = at
}
switch l := args["limit"].(type) {
case nil:
case float64:
if l < 0 || l != float64(int(l)) {
return q, fmt.Errorf("limit %v is no count", l)
}
q.Limit = int(l)
default:
return q, fmt.Errorf("limit %v is no count", l)
}
return q, nil
}
// Pipelines is view_pipelines: the kept pipelines that fit q, newest first (by their start, else when heard), each with the latest
// progress of its running steps from the progress state.
func (r Reader) Pipelines(q PipelinesQuery) (PipelinesAnswer, error) {
out := PipelinesAnswer{Pipelines: []PipelineView{}}
if r.PipelinesState.Keys == nil && r.PipelinesState.All == nil {
return out, errors.New("the pipelines state cannot be read here")
}
all, err := values(r.PipelinesState)
if err != nil {
return out, err
}
var progress map[string]json.RawMessage
if r.ProgressState.Keys != nil || r.ProgressState.All != nil {
if progress, err = values(r.ProgressState); err != nil {
return out, fmt.Errorf("the pipelines' progress cannot be read: %w", err)
}
}
var found []PipelineView
for _, key := range sortedKeys(all) {
var rec PipelineRecord
if err := json.Unmarshal(all[key], &rec); err != nil {
out.Unreadable = append(out.Unreadable, key)
continue
}
if q.Branch != "" && rec.Branch != q.Branch || q.Module != "" && rec.Module != q.Module || q.Outcome != "" && rec.Outcome != q.Outcome ||
!q.Since.IsZero() && rec.Heard.Before(q.Since) {
continue
}
pv := PipelineView{Pipeline: rec.Pipeline, Heard: rec.Heard, Steps: make([]PipelineStepView, 0, len(rec.Steps))}
for _, s := range rec.Steps {
sv := PipelineStepView{StepResult: s}
if s.Ask != "" && s.Outcome == "running" {
var p ProgressRecord
if raw := progress[ProgressKey(s.Ask)]; len(raw) > 0 && json.Unmarshal(raw, &p) == nil {
sv.Progress = &StepProgressView{Phase: p.Phase, At: p.At, Seat: p.Seat, Node: p.Node}
}
}
pv.Steps = append(pv.Steps, sv)
}
found = append(found, pv)
}
when := func(p PipelineView) time.Time {
if p.Started != nil {
return *p.Started
}
return p.Heard
}
sort.SliceStable(found, func(i, j int) bool { return when(found[i]).After(when(found[j])) })
out.Matching = len(found)
limit := q.Limit
if limit <= 0 {
limit = DefaultPipelinesLimit
}
if limit > MaxPipelinesLimit {
limit = MaxPipelinesLimit
}
if len(found) > limit {
found, out.Cut = found[:limit], true
}
if found != nil {
out.Pipelines = found
}
return out, nil
}
func sortedKeys(m map[string]json.RawMessage) []string {
ks := make([]string, 0, len(m))
for k := range m {
ks = append(ks, k)
}
sort.Strings(ks)
return ks
}