Files
mesh-delivery/modules/mesh-delivery-pipeline/internal/pipeline/steps.go
T
jschoubben 5db642e161 Merge branch 'feat/47-G-fix11-b1f' into feat/47-G-join11
# Conflicts:
#	modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main.go
#	modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline-tools/main_test.go
#	modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go
#	modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go
#	modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go
#	modules/mesh-delivery-pipeline/go.mod
#	modules/mesh-delivery-pipeline/internal/pipeline/read_test.go
#	modules/mesh-delivery-pipeline/internal/pipeline/start.go
#	modules/mesh-delivery-pipeline/manifest_test.go
#	modules/mesh-delivery-pipeline/module.json
2026-11-12 10:30:22 +01:00

401 lines
18 KiB
Go

package pipeline
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
nodebuildimage "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
nodebuildnode "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
// step-ended, on the subject of the node that ran it.
type StepSeat struct {
Seat string
// Prefix is the subject before <node>.
Prefix string
// Decode reads the seat's step-ended body; nil is the build seats' one form.
Decode func([]byte) (StepEnded, error)
}
// StepSeats are the three build seats, from their contracts.
var StepSeats = []StepSeat{
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
// The seat mesh-wants answers the composition step (§22, §24).
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
}
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
func decodeWantsEnded(raw []byte) (StepEnded, error) {
var w wants.StepEnded
if err := json.Unmarshal(raw, &w); err != nil {
return StepEnded{}, err
}
said := w.Said
if said == "" && len(w.Warnings) > 0 {
said = strings.Join(w.Warnings, "; ")
}
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
}
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
// refused (A-99).
func decodeBuildEnded(raw []byte) (StepEnded, error) {
var b builds.StepEnded
if err := json.Unmarshal(raw, &b); err != nil {
return StepEnded{}, err
}
if len(b.Entries) != 1 {
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
}
e := b.Entries[0]
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
}
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
// their definitions, so the runner reads each in this one type.
type StepEnded = nodebuildgo.StepEnded
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
var ErrRefused = errors.New("a step's end the runner does not take")
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
var ErrChanged = errors.New("the pipeline was written since it was read")
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
type Updater interface {
Get(key string) (seat.Pipeline, uint64, error)
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
// Keys are the keys matching a filter, `*` one key part.
Keys(filter string) ([]string, error)
}
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
type Hearer struct {
Store Updater
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
Say func(subject string, body any) error
Log func(seat.LogEntry) error
// LogLast is the step log's last entry of a pipeline, nil when it has none (§549).
LogLast func(pipeline string) (*seat.LogEntry, error)
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
// runner took, so a kept result is its seat holder's alone.
Keep func(key string, k seat.KeptResult) error
// Output keeps a step's output under key in the objects step-outputs.
Output func(key string, data []byte) error
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
Asker Asker
Name string
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
// its warrant (design 47 G A-89, §700).
Where func(ask string) (string, *oc.Warrant, error)
Logf func(string, ...any)
Now func() time.Time
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
Failures *Failures
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
Ager *Ager
// Fills says the stores' fill at each step's end (§366); nil says none.
Fills *Fills
// Widens computes the widens a pipeline-ended carries (§71, A-207); nil carries none.
Widens func(seat.Pipeline) []seat.Widening
}
func (h *Hearer) now() time.Time {
if h.Now != nil {
return h.Now().UTC()
}
return time.Now().UTC()
}
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
node := strings.TrimPrefix(subject, bs.Prefix)
if node == subject || node == "" || strings.Contains(node, ".") {
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
}
if ev.Node != node {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
}
for {
p, rev, err := h.Store.Get(ev.Pipeline)
if err != nil {
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
}
i := stepOfAsk(p, ev.ID)
switch {
case i < 0:
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
case p.Steps[i].Seat != bs.Seat:
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
return p, nil
case !endOutcome(ev.Outcome):
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
}
now := h.now()
s := &p.Steps[i]
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
if ev.Tests != nil {
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
}
if ev.Output != "" && h.Output != nil {
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact))
if err := h.Output(name, []byte(ev.Output)); err != nil {
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
}
s.Output = name
}
var keptKeys []string
for _, k := range ev.Kept {
keptKeys = append(keptKeys, seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)))
}
unlogged := endEntry(ev.Pipeline, *s, now, keptKeys...)
p.Unlogged = append(p.Unlogged, unlogged)
from := now
if ev.Ended != nil {
from = ev.Ended.UTC()
}
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
if err != nil {
return p, err
}
ended := endPipeline(&p, now)
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
continue
} else if err != nil {
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
}
if h.Keep != nil {
for _, k := range ev.Kept {
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline, LastRead: &now}
if k.Toolchain != nil {
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
}
if kept.Pipeline == "" {
kept.Pipeline = ev.Pipeline
}
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
}
}
}
h.Failures.StepEnded(h.Store, p, *s)
if err := h.say(seat.EventStepEnded, p, s); err != nil {
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
}
// The step's end is written and said before its log entry is appended, so a log that refuses ends no step (§486, §606).
h.appendUnlogged(ev.Pipeline)
h.Ager.Age()
h.Fills.Check()
return p, h.said(p, asked, ended)
}
}
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
if asked {
if err := h.say(seat.EventWaitHeld, p, nil); err != nil {
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
}
}
if ended {
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
}
return h.groupEnded(p)
}
return nil
}
// groupEnded says a group's one pipeline's end once every member's pipeline of the same watched branch and source revision
// ended (design 47 G §73, §332): on the subject under the group, no module part, with every member's end. A pipeline of no
// group says none; a member still running says none yet.
func (h *Hearer) groupEnded(p seat.Pipeline) error {
if p.Group == "" {
return nil
}
pkeys, err := h.Store.Keys("*.*.*")
if err != nil {
return fmt.Errorf("the members of group %s are not listed: %w", p.Group, err)
}
sort.Strings(pkeys)
var members []seat.PipelineEnded
outcome, why := seat.GroupEndedOutcomePassed, ""
var last *time.Time
for _, key := range pkeys {
m, _, err := h.Store.Get(key)
if err != nil || m.Branch != p.Branch || m.Source != p.Source || m.Group != p.Group {
continue
}
if m.Outcome == seat.PipelineOutcomeRunning {
return nil
}
members = append(members, pipelineEnded(m))
if m.Ended != nil && (last == nil || m.Ended.After(*last)) {
last = m.Ended
}
if m.Outcome != seat.PipelineOutcomePassed && outcome != seat.GroupEndedOutcomeFailed {
outcome, why = m.Outcome, m.Module+": "+m.Why
if m.Outcome == seat.PipelineOutcomeFailed {
outcome = seat.GroupEndedOutcomeFailed
}
}
}
subject := "mesh.seat." + seat.Seat + ".event." + seat.EventGroupEnded + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(p.Group)
if err := h.Say(subject, seat.GroupEnded{V: 1, Branch: p.Branch, Group: p.Group, Source: p.Source, Revision: p.Revision, Ended: last,
Outcome: outcome, Why: why, Pipelines: members}); err != nil {
return fmt.Errorf("group %s ended but %s is not said: %w", p.Group, seat.EventGroupEnded, err)
}
return nil
}
// endEntry is the step log's entry of a step's end, with the keys of the kept results it wrote or carried.
func endEntry(key string, s seat.StepResult, at time.Time, kept ...string) seat.LogEntry {
return seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
Output: s.Output, Kept: kept, At: at}
}
func (h *Hearer) logf(format string, args ...any) {
if h.Logf != nil {
h.Logf(format, args...)
}
}
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries.
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
last := p.Commit
if p.Group != "" {
last = p.Group
}
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
at := h.now()
switch event {
case seat.EventStepEnded, seat.EventStepStarted:
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
case seat.EventPipelineEnded:
e := pipelineEnded(p)
if h.Widens != nil {
e.Widens = h.Widens(p)
}
return h.Say(subject, e)
default:
return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at})
}
}
// The test budget per catalog module (design 47 G §24, §419; the test budget): at most 500 unit and 50 integration tests.
const (
budgetUnit = 500
budgetIntegration = 50
)
// testCounts is the module's test counts, the sum of its test steps' own counts; nil when no step counted any.
func testCounts(steps []seat.StepResult) *seat.TestCounts {
var t seat.TestCounts
counted := false
for _, s := range steps {
if s.Tests != nil {
t.Unit, t.Integration, counted = t.Unit+s.Tests.Unit, t.Integration+s.Tests.Integration, true
}
}
if !counted {
return nil
}
return &t
}
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
tests := testCounts(p.Steps)
return seat.PipelineEnded{Tests: tests, OverBudget: tests != nil && (tests.Unit > budgetUnit || tests.Integration > budgetIntegration),
V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started,
Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps}
}
func endOutcome(o string) bool {
switch o {
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
nodebuildgo.StepEndedOutcomeCouldNotRun:
return true
}
return false
}
func stepOfAsk(p seat.Pipeline, id string) int {
for i, s := range p.Steps {
if id != "" && s.Ask == id {
return i
}
}
return -1
}
// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else
// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step
// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the
// pipeline ended now, its end count one higher.
func endPipeline(p *seat.Pipeline, now time.Time) bool {
if p.Outcome != seat.PipelineOutcomeRunning {
return false
}
var held []string
notAsked := -1
for i, s := range p.Steps {
switch {
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
notAsked = i
continue
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
// pipeline's end (A-104).
continue
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
return false
}
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome)
}
}
if notAsked >= 0 {
if len(held) == 0 {
return false
}
p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked
}
p.Ended = &now
p.EndCount++
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
if len(held) > 0 {
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
}
return true
}