The view hears the seat mesh-pipeline-runner's pipeline-started, step-started, step-ended and pipeline-ended and the build seats' step-progress, keeps each pipeline whole in its own state pipelines (newest 500) and the latest progress per running ask in pipeline-progress (200), and view_pipelines reads them. Builds: the lead's handover of 2026-11-16 (part 1 of the delivery view through its own pipeline); design 47 G, Pipelines per module (no milestone line cited by the handover)
475 lines
16 KiB
Go
475 lines
16 KiB
Go
package view
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// The view keeps the pipeline runner's pipelines in its own state `pipelines` and the latest progress of each running ask in its
|
|
// state `pipeline-progress`, from the seat mesh-pipeline-runner's events pipeline-started, step-started, step-ended and
|
|
// pipeline-ended and the build seats' step-progress; view_pipelines reads them (design 47 G *Pipelines per module*).
|
|
|
|
// PipelinesReadTool reads the view's pipelines, each whole with its steps and the progress of the steps that run.
|
|
const PipelinesReadTool = "view_pipelines"
|
|
|
|
// MaxPipelines is how many pipelines the state `pipelines` keeps: when more are kept, the longest unheard ones go. MaxProgress is the
|
|
// same for the state `pipeline-progress`, whose entry also goes when its step ends. DefaultPipelinesLimit is how many a read gives
|
|
// when no limit is asked, MaxPipelinesLimit the most.
|
|
const (
|
|
MaxPipelines = 500
|
|
MaxProgress = 200
|
|
DefaultPipelinesLimit = 50
|
|
MaxPipelinesLimit = 500
|
|
pruneSlack = 50 // a state over its cap by less than this is not cut yet, so a replay does not read it at every new key
|
|
pipelineWrites = 5 // how often a write over a changed key is tried again before the event is left for the bus to give again
|
|
)
|
|
|
|
// The seat events the view hears for its pipelines, and the build seats whose step-progress it hears.
|
|
const (
|
|
EventStepProgress = "step-progress"
|
|
)
|
|
|
|
// PipelineSubjects are the subjects of the pipeline runner's events the view hears, each the seat contract's.
|
|
func PipelineSubjects() []string {
|
|
return []string{meshpipelinerunner.SubjectPipelineStartedAll, meshpipelinerunner.SubjectStepStartedAll,
|
|
meshpipelinerunner.SubjectStepEndedAll, meshpipelinerunner.SubjectPipelineEndedAll}
|
|
}
|
|
|
|
// ProgressSubjects are the build seats' step-progress subjects the view hears, each the seat contract's.
|
|
func ProgressSubjects() []string {
|
|
return []string{nodebuildgo.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll}
|
|
}
|
|
|
|
// PipelineRecord is the value of a pipelines key: the pipeline as the runner said it (the steps with their outcomes, why, times)
|
|
// and Heard, when the view last kept an event of it.
|
|
type PipelineRecord struct {
|
|
meshpipelinerunner.Pipeline
|
|
Heard time.Time `json:"heard"`
|
|
}
|
|
|
|
// ProgressRecord is the value of a pipeline-progress key: the latest progress said of one ask, with the seat and the node that said it.
|
|
type ProgressRecord struct {
|
|
Ask string `json:"ask"`
|
|
Asker string `json:"asker,omitempty"`
|
|
Phase string `json:"phase"`
|
|
At time.Time `json:"at"`
|
|
Seat string `json:"seat"`
|
|
Node string `json:"node,omitempty"`
|
|
}
|
|
|
|
// ErrNotKept is what a store's Get says of a key it does not hold.
|
|
var ErrNotKept = errors.New("not kept")
|
|
|
|
// PipelineStore is the view's own bucket as the pipelines are kept in it: a write over a revision is refused when the key moved
|
|
// (revision 0 creates it).
|
|
type PipelineStore interface {
|
|
Get(key string) (value []byte, revision uint64, err error)
|
|
Put(key string, value []byte, revision uint64) error
|
|
Delete(key string) error
|
|
Keys() ([]string, error)
|
|
}
|
|
|
|
// ProgressKey is the key of an ask's progress.
|
|
func ProgressKey(ask string) string { return keys.EscapeName(ask) }
|
|
|
|
// ApplyPipelineEvent keeps one of the pipeline runner's events (its subject names which) in pipelines, and the progress of an ask a
|
|
// step's end or the pipeline's end closes in progress. An event that cannot be kept is an error and the bus gives it again.
|
|
func ApplyPipelineEvent(subject string, body []byte, pipelines, progress PipelineStore, now time.Time) error {
|
|
parts := strings.Split(subject, ".")
|
|
if len(parts) < 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
|
|
return fmt.Errorf("%s is no seat event's subject", subject)
|
|
}
|
|
event := parts[4]
|
|
switch event {
|
|
case meshpipelinerunner.EventPipelineStarted:
|
|
var e meshpipelinerunner.PipelineStarted
|
|
if err := json.Unmarshal(body, &e); err != nil {
|
|
return fmt.Errorf("%s: %w", subject, err)
|
|
}
|
|
return changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
|
|
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
|
|
if e.Published != nil {
|
|
p.Published = e.Published
|
|
}
|
|
if p.Started == nil {
|
|
p.Started = e.At
|
|
}
|
|
for _, s := range e.Steps {
|
|
if stepIndex(p.Steps, s.Step, s.Artifact) < 0 {
|
|
p.Steps = append(p.Steps, meshpipelinerunner.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status})
|
|
}
|
|
}
|
|
})
|
|
case meshpipelinerunner.EventStepStarted, meshpipelinerunner.EventStepEnded:
|
|
var e meshpipelinerunner.StepEvent
|
|
if err := json.Unmarshal(body, &e); err != nil {
|
|
return fmt.Errorf("%s: %w", subject, err)
|
|
}
|
|
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
|
|
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
|
|
i := stepIndex(p.Steps, e.Result.Step, e.Result.Artifact)
|
|
switch {
|
|
case i < 0:
|
|
p.Steps = append(p.Steps, e.Result)
|
|
case p.Outcome != meshpipelinerunner.PipelineEndedOutcomeRunning && p.Outcome != "" && p.Steps[i].Outcome != "":
|
|
// The pipeline's end said every step whole: a step's event heard after it changes nothing.
|
|
default:
|
|
p.Steps[i] = e.Result
|
|
}
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if event == meshpipelinerunner.EventStepEnded && e.Result.Ask != "" {
|
|
return deleteKey(progress, ProgressKey(e.Result.Ask))
|
|
}
|
|
return nil
|
|
case meshpipelinerunner.EventPipelineEnded:
|
|
var e meshpipelinerunner.PipelineEnded
|
|
if err := json.Unmarshal(body, &e); err != nil {
|
|
return fmt.Errorf("%s: %w", subject, err)
|
|
}
|
|
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
|
|
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
|
|
p.Outcome, p.Why, p.Steps, p.Tests, p.OverBudget, p.Widens = e.Outcome, e.Why, e.Steps, e.Tests, e.OverBudget, e.Widens
|
|
p.EndCount, p.Ended = e.EndCount, e.Ended
|
|
if e.Started != nil {
|
|
p.Started = e.Started
|
|
}
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, s := range e.Steps {
|
|
if s.Ask != "" {
|
|
if err := deleteKey(progress, ProgressKey(s.Ask)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%s is no pipeline event the view hears", subject)
|
|
}
|
|
|
|
// fillPipeline puts what an event names of its pipeline into p; a pipeline first heard of by a step's event has an outcome of
|
|
// running until its end says another.
|
|
func fillPipeline(p *PipelineRecord, pipeline, module, branch, commit, group, run, fingerprint string, revision, source int64) {
|
|
q := &p.Pipeline
|
|
q.Pipeline, q.Module, q.Branch, q.Commit = pipeline, module, branch, commit
|
|
q.Group, q.Run, q.Fingerprint, q.Revision, q.Source = group, run, fingerprint, revision, source
|
|
if q.Outcome == "" {
|
|
q.Outcome = meshpipelinerunner.PipelineEndedOutcomeRunning
|
|
}
|
|
}
|
|
|
|
func stepIndex(steps []meshpipelinerunner.StepResult, step, artifact string) int {
|
|
for i, s := range steps {
|
|
if s.Step == step && s.Artifact == artifact {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// changePipeline reads a pipeline's record (a new one when none), lets change alter it and writes it over the revision it read,
|
|
// again when the key moved in between; a new key over the cap cuts the oldest heard (prune).
|
|
func changePipeline(store PipelineStore, key string, now time.Time, change func(*PipelineRecord)) error {
|
|
if strings.TrimSpace(key) == "" {
|
|
return errors.New("the event names no pipeline")
|
|
}
|
|
var err error
|
|
for try := 0; try < pipelineWrites; try++ {
|
|
var rec PipelineRecord
|
|
raw, rev, gerr := store.Get(key)
|
|
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
|
|
return gerr
|
|
}
|
|
if rev != 0 && len(raw) > 0 {
|
|
if jerr := json.Unmarshal(raw, &rec); jerr != nil {
|
|
return fmt.Errorf("the pipeline %s is kept in a form that does not read: %w", key, jerr)
|
|
}
|
|
}
|
|
change(&rec)
|
|
rec.Heard = now.UTC()
|
|
out, jerr := json.Marshal(rec)
|
|
if jerr != nil {
|
|
return jerr
|
|
}
|
|
if err = store.Put(key, out, rev); err == nil {
|
|
if rev == 0 {
|
|
return prunePipelines(store, MaxPipelines)
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("the pipeline %s moved while it was kept: %w", key, err)
|
|
}
|
|
|
|
func deleteKey(store PipelineStore, key string) error {
|
|
if err := store.Delete(key); err != nil && !errors.Is(err, ErrNotKept) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// prunePipelines cuts the pipelines to max when more than max plus the slack are kept: the ones heard longest ago go.
|
|
func prunePipelines(store PipelineStore, max int) error {
|
|
all, err := store.Keys()
|
|
if err != nil || len(all) <= max+pruneSlack {
|
|
return err
|
|
}
|
|
type kept struct {
|
|
key string
|
|
heard time.Time
|
|
}
|
|
var ks []kept
|
|
for _, k := range all {
|
|
raw, _, err := store.Get(k)
|
|
if err != nil {
|
|
if errors.Is(err, ErrNotKept) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
var h struct {
|
|
Heard time.Time `json:"heard"`
|
|
}
|
|
_ = json.Unmarshal(raw, &h)
|
|
ks = append(ks, kept{k, h.Heard})
|
|
}
|
|
sort.Slice(ks, func(i, j int) bool { return ks[i].heard.Before(ks[j].heard) })
|
|
for i := 0; i < len(ks)-max; i++ {
|
|
if err := deleteKey(store, ks[i].key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ApplyProgress keeps one build seat's step-progress (a subject mesh.seat.<seat>.event.step-progress.<node>): the latest per ask, so a
|
|
// progress older than the one kept changes nothing.
|
|
func ApplyProgress(subject string, body []byte, progress PipelineStore) error {
|
|
parts := strings.Split(subject, ".")
|
|
if len(parts) != 6 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" || parts[4] != EventStepProgress {
|
|
return fmt.Errorf("%s is no step-progress subject", subject)
|
|
}
|
|
var e struct {
|
|
Ask string `json:"ask"`
|
|
Asker string `json:"asker"`
|
|
Phase string `json:"phase"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
if err := json.Unmarshal(body, &e); err != nil {
|
|
return fmt.Errorf("%s: %w", subject, err)
|
|
}
|
|
if e.Ask == "" || e.Phase == "" {
|
|
return fmt.Errorf("%s names no ask or no phase", subject)
|
|
}
|
|
key := ProgressKey(e.Ask)
|
|
rec := ProgressRecord{Ask: e.Ask, Asker: e.Asker, Phase: e.Phase, At: e.At.UTC(), Seat: parts[2], Node: parts[5]}
|
|
out, err := json.Marshal(rec)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for try := 0; try < pipelineWrites; try++ {
|
|
raw, rev, gerr := progress.Get(key)
|
|
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
|
|
return gerr
|
|
}
|
|
if rev != 0 {
|
|
var have ProgressRecord
|
|
if json.Unmarshal(raw, &have) == nil && !have.At.Before(rec.At) {
|
|
return nil
|
|
}
|
|
}
|
|
if err = progress.Put(key, out, rev); err == nil {
|
|
if rev == 0 {
|
|
return pruneProgress(progress, MaxProgress)
|
|
}
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("the progress of %s moved while it was kept: %w", e.Ask, err)
|
|
}
|
|
|
|
// pruneProgress cuts the progress to max when more than max plus the slack are kept: the oldest said go (an ask whose end was never
|
|
// heard).
|
|
func pruneProgress(store PipelineStore, max int) error {
|
|
all, err := store.Keys()
|
|
if err != nil || len(all) <= max+pruneSlack {
|
|
return err
|
|
}
|
|
type kept struct {
|
|
key string
|
|
at time.Time
|
|
}
|
|
var ks []kept
|
|
for _, k := range all {
|
|
raw, _, err := store.Get(k)
|
|
if err != nil {
|
|
if errors.Is(err, ErrNotKept) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
var p ProgressRecord
|
|
_ = json.Unmarshal(raw, &p)
|
|
ks = append(ks, kept{k, p.At})
|
|
}
|
|
sort.Slice(ks, func(i, j int) bool { return ks[i].at.Before(ks[j].at) })
|
|
for i := 0; i < len(ks)-max; i++ {
|
|
if err := deleteKey(store, ks[i].key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PipelinesQuery is what view_pipelines asks: each field narrows, none means all.
|
|
type PipelinesQuery struct {
|
|
Branch, Module, Outcome string
|
|
Since time.Time // pipelines the view kept an event of at or after it
|
|
Limit int
|
|
}
|
|
|
|
// StepProgressView is a running step's latest progress: its phase and when it was said.
|
|
type StepProgressView struct {
|
|
Phase string `json:"phase"`
|
|
At time.Time `json:"at"`
|
|
Seat string `json:"seat"`
|
|
Node string `json:"node,omitempty"`
|
|
}
|
|
|
|
// PipelineStepView is a step as the runner said it, with the progress of the step's ask while it runs.
|
|
type PipelineStepView struct {
|
|
meshpipelinerunner.StepResult
|
|
Progress *StepProgressView `json:"progress,omitempty"`
|
|
}
|
|
|
|
// PipelineView is a pipeline whole with its steps, Heard when the view last kept an event of it.
|
|
type PipelineView struct {
|
|
meshpipelinerunner.Pipeline
|
|
Steps []PipelineStepView `json:"steps"`
|
|
Heard time.Time `json:"heard"`
|
|
}
|
|
|
|
// PipelinesAnswer is view_pipelines' answer: the pipelines newest first, Matching how many fit before the limit cut them.
|
|
type PipelinesAnswer struct {
|
|
Pipelines []PipelineView `json:"pipelines"`
|
|
Matching int `json:"matching"`
|
|
Cut bool `json:"cut,omitempty"`
|
|
// Unreadable names the keys of a state that do not read.
|
|
Unreadable []string `json:"unreadable,omitempty"`
|
|
}
|
|
|
|
// ParsePipelinesQuery reads the tool's arguments; since is an RFC 3339 time, limit a whole number.
|
|
func ParsePipelinesQuery(args map[string]any) (PipelinesQuery, error) {
|
|
var q PipelinesQuery
|
|
q.Branch, _ = args["branch"].(string)
|
|
q.Module, _ = args["module"].(string)
|
|
q.Outcome, _ = args["outcome"].(string)
|
|
if raw, _ := args["since"].(string); strings.TrimSpace(raw) != "" {
|
|
at, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw))
|
|
if err != nil {
|
|
return q, fmt.Errorf("since %q is not an RFC 3339 time: %w", raw, err)
|
|
}
|
|
q.Since = at
|
|
}
|
|
switch l := args["limit"].(type) {
|
|
case nil:
|
|
case float64:
|
|
if l < 0 || l != float64(int(l)) {
|
|
return q, fmt.Errorf("limit %v is no count", l)
|
|
}
|
|
q.Limit = int(l)
|
|
default:
|
|
return q, fmt.Errorf("limit %v is no count", l)
|
|
}
|
|
return q, nil
|
|
}
|
|
|
|
// Pipelines is view_pipelines: the kept pipelines that fit q, newest first (by their start, else when heard), each with the latest
|
|
// progress of its running steps from the progress state.
|
|
func (r Reader) Pipelines(q PipelinesQuery) (PipelinesAnswer, error) {
|
|
out := PipelinesAnswer{Pipelines: []PipelineView{}}
|
|
if r.PipelinesState.Keys == nil && r.PipelinesState.All == nil {
|
|
return out, errors.New("the pipelines state cannot be read here")
|
|
}
|
|
all, err := values(r.PipelinesState)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
var progress map[string]json.RawMessage
|
|
if r.ProgressState.Keys != nil || r.ProgressState.All != nil {
|
|
if progress, err = values(r.ProgressState); err != nil {
|
|
return out, fmt.Errorf("the pipelines' progress cannot be read: %w", err)
|
|
}
|
|
}
|
|
var found []PipelineView
|
|
for _, key := range sortedKeys(all) {
|
|
var rec PipelineRecord
|
|
if err := json.Unmarshal(all[key], &rec); err != nil {
|
|
out.Unreadable = append(out.Unreadable, key)
|
|
continue
|
|
}
|
|
if q.Branch != "" && rec.Branch != q.Branch || q.Module != "" && rec.Module != q.Module || q.Outcome != "" && rec.Outcome != q.Outcome ||
|
|
!q.Since.IsZero() && rec.Heard.Before(q.Since) {
|
|
continue
|
|
}
|
|
pv := PipelineView{Pipeline: rec.Pipeline, Heard: rec.Heard, Steps: make([]PipelineStepView, 0, len(rec.Steps))}
|
|
for _, s := range rec.Steps {
|
|
sv := PipelineStepView{StepResult: s}
|
|
if s.Ask != "" && s.Outcome == "running" {
|
|
var p ProgressRecord
|
|
if raw := progress[ProgressKey(s.Ask)]; len(raw) > 0 && json.Unmarshal(raw, &p) == nil {
|
|
sv.Progress = &StepProgressView{Phase: p.Phase, At: p.At, Seat: p.Seat, Node: p.Node}
|
|
}
|
|
}
|
|
pv.Steps = append(pv.Steps, sv)
|
|
}
|
|
found = append(found, pv)
|
|
}
|
|
when := func(p PipelineView) time.Time {
|
|
if p.Started != nil {
|
|
return *p.Started
|
|
}
|
|
return p.Heard
|
|
}
|
|
sort.SliceStable(found, func(i, j int) bool { return when(found[i]).After(when(found[j])) })
|
|
out.Matching = len(found)
|
|
limit := q.Limit
|
|
if limit <= 0 {
|
|
limit = DefaultPipelinesLimit
|
|
}
|
|
if limit > MaxPipelinesLimit {
|
|
limit = MaxPipelinesLimit
|
|
}
|
|
if len(found) > limit {
|
|
found, out.Cut = found[:limit], true
|
|
}
|
|
if found != nil {
|
|
out.Pipelines = found
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func sortedKeys(m map[string]json.RawMessage) []string {
|
|
ks := make([]string, 0, len(m))
|
|
for k := range m {
|
|
ks = append(ks, k)
|
|
}
|
|
sort.Strings(ks)
|
|
return ks
|
|
}
|