Files
mesh-delivery/modules/mesh-delivery-pipeline/internal/pipeline/steps.go
T
jschoubben 05b91a6281 C-R6: pipeline-ended names each mover's commit and widens
The seat mesh-pipeline-runner's MoverEnded gains commit and widens (regenerated, with a
fixture row), and the runner writes them from its pipeline value when it says
pipeline-ended, so the teller says a widening from the event alone.

Builds: design 47 H H-A-119, H-A-72, H-A-51
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 05:18:13 +01:00

671 lines
29 KiB
Go

package pipeline
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
)
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
// step-ended, on the subject of the node that ran it.
type StepSeat struct {
Seat string
// Prefix is the subject before <node>.
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
// a seat whose ends name none.
Shared func([]byte) []seat.Shared
}
// StepSeats are the three build seats, from their contracts.
var StepSeats = []StepSeat{
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
}
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
// A-469t): the same form, under the same name.
func wantsShared(raw []byte) []seat.Shared {
var w wants.StepEnded
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
return nil
}
out := make([]seat.Shared, 0, len(w.Shared))
for _, x := range w.Shared {
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
if x.AskedBy != nil {
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
}
out = append(out, e)
}
return out
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
func decodeWantsEnded(raw []byte) (StepEnded, error) {
var w wants.StepEnded
if err := json.Unmarshal(raw, &w); err != nil {
return StepEnded{}, err
}
said := w.Said
if said == "" && len(w.Warnings) > 0 {
said = strings.Join(w.Warnings, "; ")
}
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
}
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
// refused (A-99).
func decodeBuildEnded(raw []byte) (StepEnded, error) {
var b builds.StepEnded
if err := json.Unmarshal(raw, &b); err != nil {
return StepEnded{}, err
}
if len(b.Entries) != 1 {
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
}
e := b.Entries[0]
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
}
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
// their definitions, so the runner reads each in this one type.
type StepEnded = nodebuildgo.StepEnded
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
var ErrRefused = errors.New("a step's end the runner does not take")
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
var ErrChanged = errors.New("the pipeline was written since it was read")
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
type Updater interface {
Get(key string) (seat.Pipeline, uint64, error)
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
// Keys are the keys matching a filter, `*` one key part.
Keys(filter string) ([]string, error)
}
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
type Hearer struct {
Store Updater
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
Say func(subject string, body any) error
Log func(seat.LogEntry) error
// LogLast is the step log's last entry of a pipeline, nil when it has none (§549).
LogLast func(pipeline string) (*seat.LogEntry, error)
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
// runner took, so a kept result is its seat holder's alone.
Keep func(key string, k seat.KeptResult) error
// Output keeps a step's output under key in the objects step-outputs.
Output func(key string, data []byte) error
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
Asker Asker
Name string
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
// its warrant (design 47 G A-89, §700).
Where func(ask string) (string, *oc.Warrant, error)
Logf func(string, ...any)
Now func() time.Time
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
Failures *Failures
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
Ager *Ager
// Fills says the stores' fill at each step's end (§366); nil says none.
Fills *Fills
// Widens computes a mover's widens, kept in its value at the pipeline's end (§71, A-207; H-A-51); nil keeps none.
Widens func(p seat.Pipeline, module string) []seat.Widening
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
TestRuns TestRuns
Clock RunClock
Runs RunSettings
DropRun Drop
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
RunOutputs RunObjects
// AskSteps puts the asks a step's end made due, a mover's composition once validate passed (§745; Starter.AskSteps);
// nil asks none.
AskSteps func(key string, asks []Asked) error
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
// edge, §746; Starter.Release); nil places none.
Release func(key string)
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
// Starter.Turn); nil takes none.
Turn func(key string)
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
// nil keys none.
Rejoin func(key string)
}
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
// of a run that was never asked.
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
var first error
for _, reader := range readers {
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
first = err
}
}
return first
}
func (h *Hearer) now() time.Time {
if h.Now != nil {
return h.Now().UTC()
}
return time.Now().UTC()
}
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
return h.OnStepEndedShared(bs, subject, ev, nil)
}
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
// A-469m).
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
}
if ev.Node != node {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
}
// The end of a step another asker asked (the assigner's composer build) is not the runner's: it is read before any
// pipeline, and writes and says nothing (design 47 G §24, §69). A seat whose end names no asker is held to the store.
if ev.Asker != "" && ev.Asker != h.Name {
return seat.Pipeline{}, nil
}
for {
p, rev, err := h.Store.Get(ev.Pipeline)
if err != nil {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
}
i := stepOfAsk(p, ev.ID)
var m seat.Mover
if i >= 0 {
m, _ = MoverOf(p, p.Steps[i].Module)
}
switch {
case i < 0:
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
case p.Steps[i].Seat != bs.Seat:
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
case !open(p.Steps[i]):
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
return p, h.endRunOf(p, p.Steps[i], ev)
}
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
}
return p, nil
case !endOutcome(ev.Outcome):
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if len(shared) > 0 {
s.Shared = shared
}
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if ev.Output != "" && h.Output != nil {
// A mover's build step names no artifact: its output is kept under its module, so two movers' never meet (§746).
art := s.Artifact
if art == "" {
art = s.Module
}
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(art))
if err := h.Output(name, []byte(ev.Output)); err != nil {
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
}
s.Output = name
}
var keptKeys []string
for _, k := range ev.Kept {
keptKeys = append(keptKeys, seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)))
}
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
p.Unlogged = append(p.Unlogged, unlogged)
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
readers := h.sameRunReaders(&p, i, ev, now)
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
// asked when validate did not pass.
var due []Asked
if s.Step == seat.StepResultStepValidate {
readers = append(readers, sameValidate(&p, i, now)...)
var notAsked []int
due, notAsked = releaseCompositions(&p, now)
readers = append(readers, notAsked...)
}
from := now
if ev.Ended != nil {
from = ev.Ended.UTC()
}
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
if err != nil {
return p, err
}
// A mover's build passed is what a kind-3 edge waits for (§746, H-A-83 (2)); one that did not pass ends its
// dependents' steps not asked.
release := s.Step == seat.StepResultStepBuild && (!stepPassed(*s) || built(&p, s.Module))
ended := h.endPipeline(&p, now)
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
// finding's event the bus is slow to take) delays it past its bound (A-470o).
runErr := h.endRunOf(p, *s, ev)
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
if k.Toolchain != nil {
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
}
if kept.Pipeline == "" {
kept.Pipeline = ev.Pipeline
}
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
}
}
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
for _, j := range readers {
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
}
if release && h.Release != nil {
h.Release(ev.Pipeline)
}
if h.AskSteps != nil && len(due) > 0 {
if err := h.AskSteps(ev.Pipeline, due); err != nil {
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)
}
}
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
h.appendUnlogged(ev.Pipeline)
h.Ager.Age()
h.Fills.Check()
if err := h.said(p, asked, ended); err != nil {
return p, err
}
return p, runErr
}
}
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
// answers their indexes.
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
asker := p.Steps[i]
run := sharedRun(asker)
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
return nil
}
marks := map[string]RunReader{}
if r, _, err := h.TestRuns.Get(run); err == nil {
for _, o := range r.Readers {
if o.Pipeline == p.Pipeline {
marks[o.Artifact] = o
}
}
}
var out []int
for j := range p.Steps {
s := &p.Steps[j]
if j == i || sharedRun(*s) != run {
continue
}
switch m := marks[s.Artifact]; {
case open(*s) && s.Ask == "":
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
s.Rerun, s.By = true, m.By
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
default:
continue
}
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
if asker.Tests != nil {
t := *asker.Tests
s.Tests = &t
}
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
out = append(out, j)
}
return out
}
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
// A step that asked no run ends nothing more.
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
run := sharedRun(s)
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
return nil
}
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
if ev.Tests != nil {
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if h.RunOutputs != nil {
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
}
raw, err := json.Marshal(out)
if err == nil {
err = h.RunOutputs.Put(run, raw)
}
if err != nil {
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
}
end.Output = run
}
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall;
// an end is said once, on <branch>.<source> (design 47 H §746). The runner says no group-ended (§764).
func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
for _, i := range asked {
if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil {
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
}
}
// A pipeline waiting on the operator's approval alone does not count against pipelines-at-once (§748, H-A-106 (b)): its
// running key goes and the next waiting is placed, as at an end.
if !ended && h.Turn != nil && awaitsApprovalOnly(p) {
h.Turn(p.Pipeline)
}
if ended {
h.Failures.Ended(p)
err := h.say(seat.EventPipelineEnded, p, nil)
// Every end, a cancel's alike, places the next waiting (§748, H-A-53), said or not.
if h.Turn != nil {
h.Turn(p.Pipeline)
}
if err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
}
return nil
}
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, Kept: kept, At: at}
}
func (h *Hearer) logf(format string, args ...any) {
if h.Logf != nil {
h.Logf(format, args...)
}
}
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step event or a wait
// carries. A step's events are said on <module>.<branch>.<commit> of its mover; a pipeline's end and its waits on
// <branch>.<source> (design 47 H §746, H-A-72 (3), (4)).
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
subject, ev := stepEvent(event, p, *step, at)
return h.Say(subject, ev)
case seat.EventPipelineEnded:
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p))
default:
w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run,
Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at}
if step != nil {
m, _ := MoverOf(p, step.Module)
w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask
}
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w)
}
}
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for its mover: its result as it stands,
// with its shared when it reads a run (§729; A-468e, A-469q; H §746).
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
if event == seat.EventStepStarted {
// A step-started names the run and its ask, never the step that asked it (A-469q).
var shared []seat.Shared
for _, x := range step.Shared {
x.AskedBy = nil
shared = append(shared, x)
}
step.Shared = shared
}
m, _ := MoverOf(p, step.Module)
return runnerclient.Subject(event, m.Module, p.Branch, m.Commit), seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline,
Branch: p.Branch, Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint, Revision: p.Revision, Source: p.Source,
Run: p.Run, Result: step}
}
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
const (
budgetUnit = 500
budgetIntegration = 50
)
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
// steps read it (§726, A-466i); nil when no step counted any.
func testCounts(steps []seat.StepResult) *seat.TestCounts {
var t seat.TestCounts
counted := false
runs := map[string]bool{}
for _, s := range steps {
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
if runs[run] {
continue
}
runs[run] = true
}
if s.Tests != nil {
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
}
}
if !counted {
return nil
}
return &t
}
// stepsOf are the steps of p's mover module.
func stepsOf(p seat.Pipeline, module string) []seat.StepResult {
out := []seat.StepResult{}
for _, s := range p.Steps {
if s.Module == module {
out = append(out, s)
}
}
return out
}
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per
// mover, in the delivery order of its tiers, its module, fingerprint, commit, widens (H-A-119), released version, step results
// and tier.
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
movers := make([]seat.MoverEnded, 0, len(p.Movers))
for _, m := range p.Movers {
movers = append(movers, seat.MoverEnded{Module: m.Module, Fingerprint: m.Fingerprint, Commit: m.Commit, Widens: m.Widens, Version: m.Version,
Steps: stepsOf(p, m.Module), Tier: m.Tier})
}
sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier })
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision,
Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers}
}
func endOutcome(o string) bool {
switch o {
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
nodebuildgo.StepEndedOutcomeCouldNotRun:
return true
}
return false
}
func stepOfAsk(p seat.Pipeline, id string) int {
for i, s := range p.Steps {
if id != "" && s.Ask == id {
return i
}
}
return -1
}
// stepPassed says a required step holds nothing: it passed, was carried, or had nothing to run (design 47 H §746, ADR
// 0682 §8, H-A-3).
func stepPassed(s seat.StepResult) bool {
return s.Outcome == seat.StepResultOutcomePassed || s.Outcome == seat.StepResultOutcomeCarried || s.Outcome == seat.StepResultOutcomeNothingToRun
}
// endPipeline ends a pipeline whose every mover is placed and every step ended: passed when every mover's required steps
// passed, were carried or had nothing to run, else failed, naming each required step that held it (ADR 0666 §7, §9;
// design 47 H §746). A failed optional step, and an optional step still waiting, hold nothing. An approval step still
// waiting when a required step did not pass is `not asked` (§26). It says whether the pipeline ended now, its end count one
// higher.
func endPipeline(p *seat.Pipeline, now time.Time) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
for _, m := range p.Movers {
if !placed(*p, m.Module) {
return false
}
}
var held []string
var notAsked []int
for i, s := range p.Steps {
switch {
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
notAsked = append(notAsked, i)
continue
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
return false
}
if s.Status == seat.StepResultStatusRequired && !stepPassed(s) {
held = append(held, strings.TrimSpace(s.Module+" "+s.Artifact+" "+s.Step)+" "+s.Outcome)
}
}
if len(notAsked) > 0 {
if len(held) == 0 {
return false
}
for _, i := range notAsked {
p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked
}
}
p.Ended = &now
p.EndCount++
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
if len(held) > 0 {
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
}
for i := range p.Movers {
m := &p.Movers[i]
m.Tests = testCounts(stepsOf(*p, m.Module))
m.OverBudget = m.Tests != nil && (m.Tests.Unit > budgetUnit || m.Tests.Integration > budgetIntegration)
}
return true
}
// endPipeline is endPipeline with each mover's widens computed into its value at the end (§71, A-207; H-A-51).
func (h *Hearer) endPipeline(p *seat.Pipeline, now time.Time) bool {
if !endPipeline(p, now) {
return false
}
if h.Widens != nil {
for i := range p.Movers {
p.Movers[i].Widens = h.Widens(*p, p.Movers[i].Module)
}
}
return true
}
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
var first error
for _, i := range steps {
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
}
}
h.appendUnlogged(p.Pipeline)
if len(steps) > 0 {
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
}
if err := h.said(p, nil, true); err != nil && first == nil {
first = err
}
return first
}
// sameValidate ends, in p, every other mover's validate step reading the ask step i's end ended, with its result (§745,
// H-A-51). It answers their indexes.
func sameValidate(p *seat.Pipeline, i int, now time.Time) []int {
v := p.Steps[i]
var out []int
for j := range p.Steps {
s := &p.Steps[j]
if j == i || s.Step != seat.StepResultStepValidate || s.Ask != v.Ask || !open(*s) {
continue
}
s.Outcome, s.Why, s.Node, s.Ended = v.Outcome, v.Why, v.Node, &now
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
out = append(out, j)
}
return out
}