The seat mesh-pipeline-runner's MoverEnded gains commit and widens (regenerated, with a fixture row), and the runner writes them from its pipeline value when it says pipeline-ended, so the teller says a widening from the event alone. Builds: design 47 H H-A-119, H-A-72, H-A-51 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
671 lines
29 KiB
Go
671 lines
29 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
|
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
|
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
|
|
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
|
)
|
|
|
|
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
|
|
// step-ended, on the subject of the node that ran it.
|
|
type StepSeat struct {
|
|
Seat string
|
|
// Prefix is the subject before <node>.
|
|
Prefix string
|
|
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
|
Decode func([]byte) (StepEnded, error)
|
|
// Shared reads the runs a step-ended names in `shared`, carried into the step's result as they came (A-468d); nil for
|
|
// a seat whose ends name none.
|
|
Shared func([]byte) []seat.Shared
|
|
}
|
|
|
|
// StepSeats are the three build seats, from their contracts.
|
|
var StepSeats = []StepSeat{
|
|
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
|
|
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
|
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
|
// The seat mesh-wants answers the composition step (§22, §24).
|
|
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded, Shared: wantsShared},
|
|
}
|
|
|
|
// wantsShared is the composition runs a mesh-wants step-ended names, one per machine (design 47 G §723, §729; A-468d,
|
|
// A-469t): the same form, under the same name.
|
|
func wantsShared(raw []byte) []seat.Shared {
|
|
var w wants.StepEnded
|
|
if json.Unmarshal(raw, &w) != nil || len(w.Shared) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]seat.Shared, 0, len(w.Shared))
|
|
for _, x := range w.Shared {
|
|
e := seat.Shared{Machine: x.Machine, Run: x.Run, Ask: x.Ask}
|
|
if x.AskedBy != nil {
|
|
e.AskedBy = &seat.SharedAskedBy{Pipeline: x.AskedBy.Pipeline, Artifact: x.AskedBy.Artifact}
|
|
}
|
|
out = append(out, e)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
|
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
|
|
func decodeWantsEnded(raw []byte) (StepEnded, error) {
|
|
var w wants.StepEnded
|
|
if err := json.Unmarshal(raw, &w); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
said := w.Said
|
|
if said == "" && len(w.Warnings) > 0 {
|
|
said = strings.Join(w.Warnings, "; ")
|
|
}
|
|
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
|
|
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
|
|
}
|
|
|
|
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
|
|
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
|
|
// refused (A-99).
|
|
func decodeBuildEnded(raw []byte) (StepEnded, error) {
|
|
var b builds.StepEnded
|
|
if err := json.Unmarshal(raw, &b); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
if len(b.Entries) != 1 {
|
|
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
|
|
}
|
|
e := b.Entries[0]
|
|
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
|
|
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
|
|
}
|
|
|
|
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
|
|
// their definitions, so the runner reads each in this one type.
|
|
type StepEnded = nodebuildgo.StepEnded
|
|
|
|
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
|
|
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
|
|
var ErrRefused = errors.New("a step's end the runner does not take")
|
|
|
|
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
|
|
var ErrChanged = errors.New("the pipeline was written since it was read")
|
|
|
|
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
|
|
type Updater interface {
|
|
Get(key string) (seat.Pipeline, uint64, error)
|
|
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
|
|
// Keys are the keys matching a filter, `*` one key part.
|
|
Keys(filter string) ([]string, error)
|
|
}
|
|
|
|
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
|
|
type Hearer struct {
|
|
Store Updater
|
|
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
|
|
Say func(subject string, body any) error
|
|
Log func(seat.LogEntry) error
|
|
// LogLast is the step log's last entry of a pipeline, nil when it has none (§549).
|
|
LogLast func(pipeline string) (*seat.LogEntry, error)
|
|
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
|
|
// runner took, so a kept result is its seat holder's alone.
|
|
Keep func(key string, k seat.KeptResult) error
|
|
// Output keeps a step's output under key in the objects step-outputs.
|
|
Output func(key string, data []byte) error
|
|
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
|
|
Asker Asker
|
|
Name string
|
|
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
|
|
// its warrant (design 47 G A-89, §700).
|
|
Where func(ask string) (string, *oc.Warrant, error)
|
|
Logf func(string, ...any)
|
|
Now func() time.Time
|
|
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
|
|
Failures *Failures
|
|
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
|
|
Ager *Ager
|
|
// Fills says the stores' fill at each step's end (§366); nil says none.
|
|
Fills *Fills
|
|
// Widens computes a mover's widens, kept in its value at the pipeline's end (§71, A-207; H-A-51); nil keeps none.
|
|
Widens func(p seat.Pipeline, module string) []seat.Widening
|
|
// TestRuns is the state test-runs and Clock its bound keys (§725); Runs the settings of its runs; DropRun drops a
|
|
// run's ask when its run ends unfinished (§119). A nil TestRuns keeps no test run.
|
|
TestRuns TestRuns
|
|
Clock RunClock
|
|
Runs RunSettings
|
|
DropRun Drop
|
|
// RunOutputs keeps a test run's output in test-runs-objects (§725, A-466v).
|
|
RunOutputs RunObjects
|
|
// AskSteps puts the asks a step's end made due, a mover's composition once validate passed (§745; Starter.AskSteps);
|
|
// nil asks none.
|
|
AskSteps func(key string, asks []Asked) error
|
|
// Release places the movers of a pipeline whose awaited publishes are heard now, after a build step passed (a kind-3
|
|
// edge, §746; Starter.Release); nil places none.
|
|
Release func(key string)
|
|
// Turn takes a pipeline's end into the queue, its running key deleted and the next placed (design 47 H §748;
|
|
// Starter.Turn); nil takes none.
|
|
Turn func(key string)
|
|
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
|
|
// nil keys none.
|
|
Rejoin func(key string)
|
|
}
|
|
|
|
// EndReaders ends each of readers' steps of run with r's result (§725, §726): a step that read an ended run, or the steps
|
|
// of a run that was never asked.
|
|
func (h *Hearer) EndReaders(run string, readers []seat.TestRunReader, r seat.TestRun) error {
|
|
var first error
|
|
for _, reader := range readers {
|
|
if err := h.endReader(RunReader{Pipeline: reader.Pipeline, Artifact: reader.Artifact}, run, Run{TestRun: r}); err != nil && first == nil {
|
|
first = err
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
func (h *Hearer) now() time.Time {
|
|
if h.Now != nil {
|
|
return h.Now().UTC()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
|
|
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
|
|
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
|
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
|
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
|
return h.OnStepEndedShared(bs, subject, ev, nil)
|
|
}
|
|
|
|
// OnStepEndedShared is OnStepEnded of an end naming the runs it read in shared (A-468d): kept on the step's result. The end
|
|
// of a test run's ask also ends the run, its output kept in test-runs-objects, and every other step reading it (§725, §726,
|
|
// A-469m).
|
|
func (h *Hearer) OnStepEndedShared(bs StepSeat, subject string, ev StepEnded, shared []seat.Shared) (seat.Pipeline, error) {
|
|
node := strings.TrimPrefix(subject, bs.Prefix)
|
|
if node == subject || node == "" || strings.Contains(node, ".") {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
|
}
|
|
if ev.Node != node {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
|
|
}
|
|
// The end of a step another asker asked (the assigner's composer build) is not the runner's: it is read before any
|
|
// pipeline, and writes and says nothing (design 47 G §24, §69). A seat whose end names no asker is held to the store.
|
|
if ev.Asker != "" && ev.Asker != h.Name {
|
|
return seat.Pipeline{}, nil
|
|
}
|
|
for {
|
|
p, rev, err := h.Store.Get(ev.Pipeline)
|
|
if err != nil {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
|
|
}
|
|
i := stepOfAsk(p, ev.ID)
|
|
var m seat.Mover
|
|
if i >= 0 {
|
|
m, _ = MoverOf(p, p.Steps[i].Module)
|
|
}
|
|
switch {
|
|
case i < 0:
|
|
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
|
|
case p.Steps[i].Seat != bs.Seat:
|
|
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
|
|
case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step:
|
|
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
|
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
|
|
case !open(p.Steps[i]):
|
|
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
|
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
|
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
|
return p, h.endRunOf(p, p.Steps[i], ev)
|
|
}
|
|
// The run ended before its ask's end came (its bound, a drop): the end is ignored and said (§720, §725).
|
|
h.logf("the step-ended of ask %s is ignored: its test run %s ended before it (%s)", ev.ID, run, p.Steps[i].Why)
|
|
}
|
|
return p, nil
|
|
case !endOutcome(ev.Outcome):
|
|
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
|
|
}
|
|
now := h.now()
|
|
s := &p.Steps[i]
|
|
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
|
if len(shared) > 0 {
|
|
s.Shared = shared
|
|
}
|
|
if ev.Tests != nil {
|
|
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
|
}
|
|
if ev.Output != "" && h.Output != nil {
|
|
// A mover's build step names no artifact: its output is kept under its module, so two movers' never meet (§746).
|
|
art := s.Artifact
|
|
if art == "" {
|
|
art = s.Module
|
|
}
|
|
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(art))
|
|
if err := h.Output(name, []byte(ev.Output)); err != nil {
|
|
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
|
|
}
|
|
s.Output = name
|
|
}
|
|
var keptKeys []string
|
|
for _, k := range ev.Kept {
|
|
keptKeys = append(keptKeys, seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)))
|
|
}
|
|
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
|
|
p.Unlogged = append(p.Unlogged, unlogged)
|
|
// The steps of this pipeline reading the run end in the same write, so the pipeline ends once (§726, §727).
|
|
readers := h.sameRunReaders(&p, i, ev, now)
|
|
// Every mover's validate reads the one run (§745, H-A-51); its end asks each mover's composition, or ends it not
|
|
// asked when validate did not pass.
|
|
var due []Asked
|
|
if s.Step == seat.StepResultStepValidate {
|
|
readers = append(readers, sameValidate(&p, i, now)...)
|
|
var notAsked []int
|
|
due, notAsked = releaseCompositions(&p, now)
|
|
readers = append(readers, notAsked...)
|
|
}
|
|
from := now
|
|
if ev.Ended != nil {
|
|
from = ev.Ended.UTC()
|
|
}
|
|
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
// A mover's build passed is what a kind-3 edge waits for (§746, H-A-83 (2)); one that did not pass ends its
|
|
// dependents' steps not asked.
|
|
release := s.Step == seat.StepResultStepBuild && (!stepPassed(*s) || built(&p, s.Module))
|
|
ended := h.endPipeline(&p, now)
|
|
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
|
continue
|
|
} else if err != nil {
|
|
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
|
}
|
|
// The run's end is written as its step-ended is heard, before any finding the step says, so no wait elsewhere (a
|
|
// finding's event the bus is slow to take) delays it past its bound (A-470o).
|
|
runErr := h.endRunOf(p, *s, ev)
|
|
if h.Keep != nil {
|
|
for _, k := range ev.Kept {
|
|
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
|
|
if k.Toolchain != nil {
|
|
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
|
|
}
|
|
if kept.Pipeline == "" {
|
|
kept.Pipeline = ev.Pipeline
|
|
}
|
|
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
|
|
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
|
|
}
|
|
}
|
|
}
|
|
h.Failures.StepEnded(h.Store, p, *s)
|
|
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
|
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
|
}
|
|
for _, j := range readers {
|
|
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
|
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
|
}
|
|
}
|
|
if release && h.Release != nil {
|
|
h.Release(ev.Pipeline)
|
|
}
|
|
if h.AskSteps != nil && len(due) > 0 {
|
|
if err := h.AskSteps(ev.Pipeline, due); err != nil {
|
|
h.logf("the compositions of %s are written running and not asked, asked again at the next start: %v", ev.Pipeline, err)
|
|
}
|
|
}
|
|
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
|
|
h.appendUnlogged(ev.Pipeline)
|
|
h.Ager.Age()
|
|
h.Fills.Check()
|
|
if err := h.said(p, asked, ended); err != nil {
|
|
return p, err
|
|
}
|
|
return p, runErr
|
|
}
|
|
}
|
|
|
|
// sameRunReaders ends, in p, every other step reading the test run step i asked with ev, its ask's end: a running reader,
|
|
// and one the rerun that asked the run marked, which moves (A-470d, A-470e), said again marked a rerun by that caller. It
|
|
// answers their indexes.
|
|
func (h *Hearer) sameRunReaders(p *seat.Pipeline, i int, ev StepEnded, now time.Time) []int {
|
|
asker := p.Steps[i]
|
|
run := sharedRun(asker)
|
|
if run == "" || h.TestRuns == nil || asker.Ask != ev.ID {
|
|
return nil
|
|
}
|
|
marks := map[string]RunReader{}
|
|
if r, _, err := h.TestRuns.Get(run); err == nil {
|
|
for _, o := range r.Readers {
|
|
if o.Pipeline == p.Pipeline {
|
|
marks[o.Artifact] = o
|
|
}
|
|
}
|
|
}
|
|
var out []int
|
|
for j := range p.Steps {
|
|
s := &p.Steps[j]
|
|
if j == i || sharedRun(*s) != run {
|
|
continue
|
|
}
|
|
switch m := marks[s.Artifact]; {
|
|
case open(*s) && s.Ask == "":
|
|
case m.Rerun && s.Shared[0].Ask != ev.ID && s.Accepted == nil:
|
|
s.Rerun, s.By = true, m.By
|
|
s.Shared = []seat.Shared{{Run: run, Ask: ev.ID, AskedBy: &seat.SharedAskedBy{Pipeline: p.Pipeline, Artifact: asker.Artifact}}}
|
|
default:
|
|
continue
|
|
}
|
|
s.Outcome, s.Why, s.Node, s.Ended, s.Tests = asker.Outcome, asker.Why, asker.Node, &now, nil
|
|
if asker.Tests != nil {
|
|
t := *asker.Tests
|
|
s.Tests = &t
|
|
}
|
|
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
|
|
out = append(out, j)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// endRunOf ends the test run step s of p asked, with the end ev of its ask (§725, §726): the run's output kept in
|
|
// test-runs-objects under the run's key, its outcome, words, counts and node written, and every other reader ended with it.
|
|
// A step that asked no run ends nothing more.
|
|
func (h *Hearer) endRunOf(p seat.Pipeline, s seat.StepResult, ev StepEnded) error {
|
|
run := sharedRun(s)
|
|
if run == "" || h.TestRuns == nil || s.Ask != ev.ID {
|
|
return nil
|
|
}
|
|
end := seat.TestRun{Ask: ev.ID, Outcome: ev.Outcome, Said: ev.Said, Node: ev.Node}
|
|
if ev.Tests != nil {
|
|
end.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
|
}
|
|
if h.RunOutputs != nil {
|
|
// The run's output object holds the packages it ran with their digests and the build seat's output (§725).
|
|
out := RunOutput{Output: ev.Output, Packages: []seat.TestRunPackage{}}
|
|
for _, k := range append(append([]nodebuildgo.KeptResult(nil), ev.Kept...), ev.Carried...) {
|
|
out.Packages = append(out.Packages, seat.TestRunPackage{Package: k.Package, Digest: k.Digest})
|
|
}
|
|
raw, err := json.Marshal(out)
|
|
if err == nil {
|
|
err = h.RunOutputs.Put(run, raw)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("the output of the test run %s cannot be kept: %w", run, err)
|
|
}
|
|
end.Output = run
|
|
}
|
|
return h.EndRun(run, end, &seat.TestRunReader{Pipeline: p.Pipeline, Artifact: s.Artifact})
|
|
}
|
|
|
|
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall;
|
|
// an end is said once, on <branch>.<source> (design 47 H §746). The runner says no group-ended (§764).
|
|
func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
|
for _, i := range asked {
|
|
if err := h.say(seat.EventWaitHeld, p, &p.Steps[i]); err != nil {
|
|
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
|
|
}
|
|
}
|
|
// A pipeline waiting on the operator's approval alone does not count against pipelines-at-once (§748, H-A-106 (b)): its
|
|
// running key goes and the next waiting is placed, as at an end.
|
|
if !ended && h.Turn != nil && awaitsApprovalOnly(p) {
|
|
h.Turn(p.Pipeline)
|
|
}
|
|
if ended {
|
|
h.Failures.Ended(p)
|
|
err := h.say(seat.EventPipelineEnded, p, nil)
|
|
// Every end, a cancel's alike, places the next waiting (§748, H-A-53), said or not.
|
|
if h.Turn != nil {
|
|
h.Turn(p.Pipeline)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
|
|
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
|
|
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
|
Output: s.Output, Kept: kept, At: at}
|
|
}
|
|
|
|
func (h *Hearer) logf(format string, args ...any) {
|
|
if h.Logf != nil {
|
|
h.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step event or a wait
|
|
// carries. A step's events are said on <module>.<branch>.<commit> of its mover; a pipeline's end and its waits on
|
|
// <branch>.<source> (design 47 H §746, H-A-72 (3), (4)).
|
|
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
|
|
at := h.now()
|
|
switch event {
|
|
case seat.EventStepEnded, seat.EventStepStarted:
|
|
subject, ev := stepEvent(event, p, *step, at)
|
|
return h.Say(subject, ev)
|
|
case seat.EventPipelineEnded:
|
|
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), pipelineEnded(p))
|
|
default:
|
|
w := seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Revision: p.Revision, Source: p.Source, Run: p.Run,
|
|
Step: seat.WaitStepApproval, On: seat.WaitOnPerson, Since: at}
|
|
if step != nil {
|
|
m, _ := MoverOf(p, step.Module)
|
|
w.Module, w.Commit, w.Fingerprint, w.Step, w.Ask = m.Module, m.Commit, m.Fingerprint, step.Step, step.Ask
|
|
}
|
|
return h.Say(runnerclient.SourceSubject(event, p.Branch, p.Source), w)
|
|
}
|
|
}
|
|
|
|
// stepEvent is a step's step-started or step-ended of p, on the runner's subject for its mover: its result as it stands,
|
|
// with its shared when it reads a run (§729; A-468e, A-469q; H §746).
|
|
func stepEvent(event string, p seat.Pipeline, step seat.StepResult, at time.Time) (string, seat.StepEvent) {
|
|
if event == seat.EventStepStarted {
|
|
// A step-started names the run and its ask, never the step that asked it (A-469q).
|
|
var shared []seat.Shared
|
|
for _, x := range step.Shared {
|
|
x.AskedBy = nil
|
|
shared = append(shared, x)
|
|
}
|
|
step.Shared = shared
|
|
}
|
|
m, _ := MoverOf(p, step.Module)
|
|
return runnerclient.Subject(event, m.Module, p.Branch, m.Commit), seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline,
|
|
Branch: p.Branch, Module: m.Module, Commit: m.Commit, Fingerprint: m.Fingerprint, Revision: p.Revision, Source: p.Source,
|
|
Run: p.Run, Result: step}
|
|
}
|
|
|
|
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
|
|
const (
|
|
budgetUnit = 500
|
|
budgetIntegration = 50
|
|
)
|
|
|
|
// testCounts is the module's test counts, the sum of its test steps' own counts, each test run counted once however many
|
|
// steps read it (§726, A-466i); nil when no step counted any.
|
|
func testCounts(steps []seat.StepResult) *seat.TestCounts {
|
|
var t seat.TestCounts
|
|
counted := false
|
|
runs := map[string]bool{}
|
|
for _, s := range steps {
|
|
if run := sharedRun(s); run != "" && s.Seat != wants.Seat {
|
|
if runs[run] {
|
|
continue
|
|
}
|
|
runs[run] = true
|
|
}
|
|
if s.Tests != nil {
|
|
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
|
|
}
|
|
}
|
|
if !counted {
|
|
return nil
|
|
}
|
|
return &t
|
|
}
|
|
|
|
// stepsOf are the steps of p's mover module.
|
|
func stepsOf(p seat.Pipeline, module string) []seat.StepResult {
|
|
out := []seat.StepResult{}
|
|
for _, s := range p.Steps {
|
|
if s.Module == module {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// pipelineEnded is p's pipeline-ended (design 47 H §746): its key, branch, source, call, revision and outcome, and per
|
|
// mover, in the delivery order of its tiers, its module, fingerprint, commit, widens (H-A-119), released version, step results
|
|
// and tier.
|
|
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
|
|
movers := make([]seat.MoverEnded, 0, len(p.Movers))
|
|
for _, m := range p.Movers {
|
|
movers = append(movers, seat.MoverEnded{Module: m.Module, Fingerprint: m.Fingerprint, Commit: m.Commit, Widens: m.Widens, Version: m.Version,
|
|
Steps: stepsOf(p, m.Module), Tier: m.Tier})
|
|
}
|
|
sort.SliceStable(movers, func(i, j int) bool { return movers[i].Tier < movers[j].Tier })
|
|
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Source: p.Source, Call: p.Call, Revision: p.Revision,
|
|
Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started, Ended: p.Ended, EndCount: p.EndCount, Movers: movers}
|
|
}
|
|
|
|
func endOutcome(o string) bool {
|
|
switch o {
|
|
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
|
|
nodebuildgo.StepEndedOutcomeCouldNotRun:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stepOfAsk(p seat.Pipeline, id string) int {
|
|
for i, s := range p.Steps {
|
|
if id != "" && s.Ask == id {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// stepPassed says a required step holds nothing: it passed, was carried, or had nothing to run (design 47 H §746, ADR
|
|
// 0682 §8, H-A-3).
|
|
func stepPassed(s seat.StepResult) bool {
|
|
return s.Outcome == seat.StepResultOutcomePassed || s.Outcome == seat.StepResultOutcomeCarried || s.Outcome == seat.StepResultOutcomeNothingToRun
|
|
}
|
|
|
|
// endPipeline ends a pipeline whose every mover is placed and every step ended: passed when every mover's required steps
|
|
// passed, were carried or had nothing to run, else failed, naming each required step that held it (ADR 0666 §7, §9;
|
|
// design 47 H §746). A failed optional step, and an optional step still waiting, hold nothing. An approval step still
|
|
// waiting when a required step did not pass is `not asked` (§26). It says whether the pipeline ended now, its end count one
|
|
// higher.
|
|
func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
|
if p.Outcome != seat.PipelineOutcomeRunning {
|
|
return false
|
|
}
|
|
for _, m := range p.Movers {
|
|
if !placed(*p, m.Module) {
|
|
return false
|
|
}
|
|
}
|
|
var held []string
|
|
var notAsked []int
|
|
for i, s := range p.Steps {
|
|
switch {
|
|
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
|
notAsked = append(notAsked, i)
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
|
|
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
|
|
// pipeline's end (A-104).
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting, open(s):
|
|
return false
|
|
}
|
|
if s.Status == seat.StepResultStatusRequired && !stepPassed(s) {
|
|
held = append(held, strings.TrimSpace(s.Module+" "+s.Artifact+" "+s.Step)+" "+s.Outcome)
|
|
}
|
|
}
|
|
if len(notAsked) > 0 {
|
|
if len(held) == 0 {
|
|
return false
|
|
}
|
|
for _, i := range notAsked {
|
|
p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked
|
|
}
|
|
}
|
|
p.Ended = &now
|
|
p.EndCount++
|
|
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
|
|
if len(held) > 0 {
|
|
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
|
|
}
|
|
for i := range p.Movers {
|
|
m := &p.Movers[i]
|
|
m.Tests = testCounts(stepsOf(*p, m.Module))
|
|
m.OverBudget = m.Tests != nil && (m.Tests.Unit > budgetUnit || m.Tests.Integration > budgetIntegration)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// endPipeline is endPipeline with each mover's widens computed into its value at the end (§71, A-207; H-A-51).
|
|
func (h *Hearer) endPipeline(p *seat.Pipeline, now time.Time) bool {
|
|
if !endPipeline(p, now) {
|
|
return false
|
|
}
|
|
if h.Widens != nil {
|
|
for i := range p.Movers {
|
|
p.Movers[i].Widens = h.Widens(*p, p.Movers[i].Module)
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// EndedAtPlacing says a pipeline its starter ended at placing, a seat with no holder (A-470g, A-470h): the step-ended of each
|
|
// step of steps, its step log entries appended, the head's finding pipeline-failed (§33), then its pipeline-ended.
|
|
func (h *Hearer) EndedAtPlacing(p seat.Pipeline, steps []int) error {
|
|
var first error
|
|
for _, i := range steps {
|
|
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil && first == nil {
|
|
first = fmt.Errorf("pipeline %s's %s is written but not said: %w", p.Pipeline, seat.EventStepEnded, err)
|
|
}
|
|
}
|
|
h.appendUnlogged(p.Pipeline)
|
|
if len(steps) > 0 {
|
|
h.Failures.StepEnded(h.Store, p, p.Steps[steps[0]])
|
|
}
|
|
if err := h.said(p, nil, true); err != nil && first == nil {
|
|
first = err
|
|
}
|
|
return first
|
|
}
|
|
|
|
// sameValidate ends, in p, every other mover's validate step reading the ask step i's end ended, with its result (§745,
|
|
// H-A-51). It answers their indexes.
|
|
func sameValidate(p *seat.Pipeline, i int, now time.Time) []int {
|
|
v := p.Steps[i]
|
|
var out []int
|
|
for j := range p.Steps {
|
|
s := &p.Steps[j]
|
|
if j == i || s.Step != seat.StepResultStepValidate || s.Ask != v.Ask || !open(*s) {
|
|
continue
|
|
}
|
|
s.Outcome, s.Why, s.Node, s.Ended = v.Outcome, v.Why, v.Node, &now
|
|
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *s, now))
|
|
out = append(out, j)
|
|
}
|
|
return out
|
|
}
|