# Conflicts: # modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go # modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go # modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go # modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go # modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go # modules/mesh-delivery-pipeline/go.mod # modules/mesh-delivery-pipeline/internal/pipeline/read_test.go # modules/mesh-delivery-pipeline/internal/pipeline/start.go # modules/mesh-delivery-pipeline/manifest_test.go # modules/mesh-delivery-pipeline/module.json
401 lines
18 KiB
Go
401 lines
18 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
|
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
|
|
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
|
|
// step-ended, on the subject of the node that ran it.
|
|
type StepSeat struct {
|
|
Seat string
|
|
// Prefix is the subject before <node>.
|
|
Prefix string
|
|
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
|
Decode func([]byte) (StepEnded, error)
|
|
}
|
|
|
|
// StepSeats are the three build seats, from their contracts.
|
|
var StepSeats = []StepSeat{
|
|
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
|
|
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
|
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
|
// The seat mesh-wants answers the composition step (§22, §24).
|
|
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
|
|
}
|
|
|
|
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
|
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
|
|
func decodeWantsEnded(raw []byte) (StepEnded, error) {
|
|
var w wants.StepEnded
|
|
if err := json.Unmarshal(raw, &w); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
said := w.Said
|
|
if said == "" && len(w.Warnings) > 0 {
|
|
said = strings.Join(w.Warnings, "; ")
|
|
}
|
|
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
|
|
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
|
|
}
|
|
|
|
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
|
|
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
|
|
// refused (A-99).
|
|
func decodeBuildEnded(raw []byte) (StepEnded, error) {
|
|
var b builds.StepEnded
|
|
if err := json.Unmarshal(raw, &b); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
if len(b.Entries) != 1 {
|
|
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
|
|
}
|
|
e := b.Entries[0]
|
|
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
|
|
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
|
|
}
|
|
|
|
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
|
|
// their definitions, so the runner reads each in this one type.
|
|
type StepEnded = nodebuildgo.StepEnded
|
|
|
|
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
|
|
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
|
|
var ErrRefused = errors.New("a step's end the runner does not take")
|
|
|
|
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
|
|
var ErrChanged = errors.New("the pipeline was written since it was read")
|
|
|
|
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
|
|
type Updater interface {
|
|
Get(key string) (seat.Pipeline, uint64, error)
|
|
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
|
|
// Keys are the keys matching a filter, `*` one key part.
|
|
Keys(filter string) ([]string, error)
|
|
}
|
|
|
|
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
|
|
type Hearer struct {
|
|
Store Updater
|
|
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
|
|
Say func(subject string, body any) error
|
|
Log func(seat.LogEntry) error
|
|
// LogLast is the step log's last entry of a pipeline, nil when it has none (§549).
|
|
LogLast func(pipeline string) (*seat.LogEntry, error)
|
|
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
|
|
// runner took, so a kept result is its seat holder's alone.
|
|
Keep func(key string, k seat.KeptResult) error
|
|
// Output keeps a step's output under key in the objects step-outputs.
|
|
Output func(key string, data []byte) error
|
|
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
|
|
Asker Asker
|
|
Name string
|
|
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
|
|
// its warrant (design 47 G A-89, §700).
|
|
Where func(ask string) (string, *oc.Warrant, error)
|
|
Logf func(string, ...any)
|
|
Now func() time.Time
|
|
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
|
|
Failures *Failures
|
|
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
|
|
Ager *Ager
|
|
// Fills says the stores' fill at each step's end (§366); nil says none.
|
|
Fills *Fills
|
|
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
|
|
Widens func(seat.Pipeline) []seat.Widening
|
|
}
|
|
|
|
func (h *Hearer) now() time.Time {
|
|
if h.Now != nil {
|
|
return h.Now().UTC()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
|
|
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
|
|
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
|
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
|
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
|
node := strings.TrimPrefix(subject, bs.Prefix)
|
|
if node == subject || node == "" || strings.Contains(node, ".") {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
|
}
|
|
if ev.Node != node {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
|
|
}
|
|
for {
|
|
p, rev, err := h.Store.Get(ev.Pipeline)
|
|
if err != nil {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
|
|
}
|
|
i := stepOfAsk(p, ev.ID)
|
|
switch {
|
|
case i < 0:
|
|
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
|
|
case p.Steps[i].Seat != bs.Seat:
|
|
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
|
|
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
|
|
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
|
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
|
|
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
|
|
return p, nil
|
|
case !endOutcome(ev.Outcome):
|
|
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
|
|
}
|
|
now := h.now()
|
|
s := &p.Steps[i]
|
|
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
|
if ev.Tests != nil {
|
|
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
|
}
|
|
if ev.Output != "" && h.Output != nil {
|
|
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact))
|
|
if err := h.Output(name, []byte(ev.Output)); err != nil {
|
|
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
|
|
}
|
|
s.Output = name
|
|
}
|
|
var keptKeys []string
|
|
for _, k := range ev.Kept {
|
|
keptKeys = append(keptKeys, seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)))
|
|
}
|
|
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
|
|
p.Unlogged = append(p.Unlogged, unlogged)
|
|
from := now
|
|
if ev.Ended != nil {
|
|
from = ev.Ended.UTC()
|
|
}
|
|
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
ended := endPipeline(&p, now)
|
|
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
|
continue
|
|
} else if err != nil {
|
|
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
|
}
|
|
if h.Keep != nil {
|
|
for _, k := range ev.Kept {
|
|
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
|
|
if k.Toolchain != nil {
|
|
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
|
|
}
|
|
if kept.Pipeline == "" {
|
|
kept.Pipeline = ev.Pipeline
|
|
}
|
|
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
|
|
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
|
|
}
|
|
}
|
|
}
|
|
h.Failures.StepEnded(h.Store, p, *s)
|
|
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
|
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
|
}
|
|
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
|
|
h.appendUnlogged(ev.Pipeline)
|
|
h.Ager.Age()
|
|
h.Fills.Check()
|
|
return p, h.said(p, asked, ended)
|
|
}
|
|
}
|
|
|
|
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
|
|
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
|
|
if asked {
|
|
if err := h.say(seat.EventWaitHeld, p, nil); err != nil {
|
|
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
|
|
}
|
|
}
|
|
if ended {
|
|
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
|
|
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
|
|
}
|
|
return h.groupEnded(p)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// groupEnded says a group's one pipeline's end once every member's pipeline of the same watched branch and source revision
|
|
// ended (design 47 G §73, §332): on the subject under the group, no module part, with every member's end. A pipeline of no
|
|
// group says none; a member still running says none yet.
|
|
func (h *Hearer) groupEnded(p seat.Pipeline) error {
|
|
if p.Group == "" {
|
|
return nil
|
|
}
|
|
pkeys, err := h.Store.Keys("*.*.*")
|
|
if err != nil {
|
|
return fmt.Errorf("the members of group %s are not listed: %w", p.Group, err)
|
|
}
|
|
sort.Strings(pkeys)
|
|
var members []seat.PipelineEnded
|
|
outcome, why := seat.GroupEndedOutcomePassed, ""
|
|
var last *time.Time
|
|
for _, key := range pkeys {
|
|
m, _, err := h.Store.Get(key)
|
|
if err != nil || m.Branch != p.Branch || m.Source != p.Source || m.Group != p.Group {
|
|
continue
|
|
}
|
|
if m.Outcome == seat.PipelineOutcomeRunning {
|
|
return nil
|
|
}
|
|
members = append(members, pipelineEnded(m))
|
|
if m.Ended != nil && (last == nil || m.Ended.After(*last)) {
|
|
last = m.Ended
|
|
}
|
|
if m.Outcome != seat.PipelineOutcomePassed && outcome != seat.GroupEndedOutcomeFailed {
|
|
outcome, why = m.Outcome, m.Module+": "+m.Why
|
|
if m.Outcome == seat.PipelineOutcomeFailed {
|
|
outcome = seat.GroupEndedOutcomeFailed
|
|
}
|
|
}
|
|
}
|
|
subject := "mesh.seat." + seat.Seat + ".event." + seat.EventGroupEnded + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(p.Group)
|
|
if err := h.Say(subject, seat.GroupEnded{V: 1, Branch: p.Branch, Group: p.Group, Source: p.Source, Revision: p.Revision, Ended: last,
|
|
Outcome: outcome, Why: why, Pipelines: members}); err != nil {
|
|
return fmt.Errorf("group %s ended but %s is not said: %w", p.Group, seat.EventGroupEnded, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
|
|
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
|
|
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
|
Output: s.Output, Kept: kept, At: at}
|
|
}
|
|
|
|
func (h *Hearer) logf(format string, args ...any) {
|
|
if h.Logf != nil {
|
|
h.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries.
|
|
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
|
|
last := p.Commit
|
|
if p.Group != "" {
|
|
last = p.Group
|
|
}
|
|
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
|
|
at := h.now()
|
|
switch event {
|
|
case seat.EventStepEnded, seat.EventStepStarted:
|
|
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
|
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
|
|
case seat.EventPipelineEnded:
|
|
e := pipelineEnded(p)
|
|
if h.Widens != nil {
|
|
e.Widens = h.Widens(p)
|
|
}
|
|
return h.Say(subject, e)
|
|
default:
|
|
return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
|
|
Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at})
|
|
}
|
|
}
|
|
|
|
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
|
|
const (
|
|
budgetUnit = 500
|
|
budgetIntegration = 50
|
|
)
|
|
|
|
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
|
|
func testCounts(steps []seat.StepResult) *seat.TestCounts {
|
|
var t seat.TestCounts
|
|
counted := false
|
|
for _, s := range steps {
|
|
if s.Tests != nil {
|
|
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
|
|
}
|
|
}
|
|
if !counted {
|
|
return nil
|
|
}
|
|
return &t
|
|
}
|
|
|
|
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
|
|
tests := testCounts(p.Steps)
|
|
return seat.PipelineEnded{Tests: tests, OverBudget: tests != nil && (tests.Unit > budgetUnit || tests.Integration > budgetIntegration),
|
|
V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
|
|
Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started,
|
|
Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps}
|
|
}
|
|
|
|
func endOutcome(o string) bool {
|
|
switch o {
|
|
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
|
|
nodebuildgo.StepEndedOutcomeCouldNotRun:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stepOfAsk(p seat.Pipeline, id string) int {
|
|
for i, s := range p.Steps {
|
|
if id != "" && s.Ask == id {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else
|
|
// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step
|
|
// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the
|
|
// pipeline ended now, its end count one higher.
|
|
func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
|
if p.Outcome != seat.PipelineOutcomeRunning {
|
|
return false
|
|
}
|
|
var held []string
|
|
notAsked := -1
|
|
for i, s := range p.Steps {
|
|
switch {
|
|
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
|
notAsked = i
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
|
|
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
|
|
// pipeline's end (A-104).
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
|
|
return false
|
|
}
|
|
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
|
held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome)
|
|
}
|
|
}
|
|
if notAsked >= 0 {
|
|
if len(held) == 0 {
|
|
return false
|
|
}
|
|
p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked
|
|
}
|
|
p.Ended = &now
|
|
p.EndCount++
|
|
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
|
|
if len(held) > 0 {
|
|
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
|
|
}
|
|
return true
|
|
}
|