A review ends a placed review step with its proven caller; rerun asks a step again under a new ask saying it is a rerun and by whom; accept takes a failed optional step; cancel ends a running pipeline cancelled and drops its asks' bounds. Each writes `pipelines` by compare-and-set, says step-ended, step-started or pipeline-ended, and records its hand act first, through AnswerSeat. Builds: design 47 G §22, §70, §482, §30, §325, §394, A-107 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
314 lines
14 KiB
Go
314 lines
14 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
|
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
|
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
|
|
wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
|
|
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
|
nodebuildimage "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
|
|
nodebuildnode "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// StepSeat is a seat whose holder runs the lint and test steps (design 47 G §22, §24): where the runner hears its
|
|
// step-ended, on the subject of the node that ran it.
|
|
type StepSeat struct {
|
|
Seat string
|
|
// Prefix is the subject before <node>.
|
|
Prefix string
|
|
// Decode reads the seat's step-ended body; nil is the build seats' one form.
|
|
Decode func([]byte) (StepEnded, error)
|
|
}
|
|
|
|
// StepSeats are the three build seats, from their contracts.
|
|
var StepSeats = []StepSeat{
|
|
{Seat: nodebuildgo.Seat, Prefix: strings.TrimSuffix(nodebuildgo.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildnode.Seat, Prefix: strings.TrimSuffix(nodebuildnode.SubjectStepEndedAll, "*")},
|
|
{Seat: nodebuildimage.Seat, Prefix: strings.TrimSuffix(nodebuildimage.SubjectStepEndedAll, "*")},
|
|
// The seat mesh-builds answers the build step of a candidate, one entry named by its fingerprint (A-97, A-99).
|
|
{Seat: builds.Seat, Prefix: strings.TrimSuffix(builds.SubjectStepEndedAll, "*"), Decode: decodeBuildEnded},
|
|
// The seat mesh-wants answers the composition step (§22, §24).
|
|
{Seat: wants.Seat, Prefix: strings.TrimSuffix(wants.SubjectStepEndedAll, "*"), Decode: decodeWantsEnded},
|
|
}
|
|
|
|
// decodeWantsEnded reads the composition step's end of the seat mesh-wants as the form the hearer holds each end to; a
|
|
// warning of a node asleep (a pass with a warning, ADR 0593 §9) is the step's `why` when the end says nothing else.
|
|
func decodeWantsEnded(raw []byte) (StepEnded, error) {
|
|
var w wants.StepEnded
|
|
if err := json.Unmarshal(raw, &w); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
said := w.Said
|
|
if said == "" && len(w.Warnings) > 0 {
|
|
said = strings.Join(w.Warnings, "; ")
|
|
}
|
|
return StepEnded{V: w.V, ID: w.ID, Pipeline: w.Pipeline, Step: w.Step, Module: w.Module, Commit: w.Commit, Fingerprint: w.Fingerprint,
|
|
Node: w.Node, Outcome: w.Outcome, Said: said, Rerun: w.Rerun, By: w.By, Started: w.Started, Ended: w.Ended}, nil
|
|
}
|
|
|
|
// decodeBuildEnded reads the build step's end of the seat mesh-builds as the form the hearer holds each end to: its one
|
|
// entry names the module, the commit and the fingerprint the step was asked at; an end naming another number of entries is
|
|
// refused (A-99).
|
|
func decodeBuildEnded(raw []byte) (StepEnded, error) {
|
|
var b builds.StepEnded
|
|
if err := json.Unmarshal(raw, &b); err != nil {
|
|
return StepEnded{}, err
|
|
}
|
|
if len(b.Entries) != 1 {
|
|
return StepEnded{}, fmt.Errorf("%w: ask %s ended a build of %d entries, asked of one", ErrRefused, b.ID, len(b.Entries))
|
|
}
|
|
e := b.Entries[0]
|
|
return StepEnded{V: b.V, ID: b.ID, Pipeline: b.Pipeline, Step: b.Step, Module: e.Module, Commit: e.Commit, Fingerprint: e.Fingerprint,
|
|
Node: b.Node, Outcome: b.Outcome, Said: b.Said, Rerun: b.Rerun, By: b.By, Started: b.Started, Ended: b.Ended}, nil
|
|
}
|
|
|
|
// StepEnded is a build seat's step-ended. The three seats' bodies are one form, the part build-step's, held equal by
|
|
// their definitions, so the runner reads each in this one type.
|
|
type StepEnded = nodebuildgo.StepEnded
|
|
|
|
// ErrRefused is a step-ended the runner does not take: not the node's holder's subject, or naming an ask, a commit or a
|
|
// fingerprint the runner did not ask (ADR 0666 §21; design 47 G §24). It is said, never taken.
|
|
var ErrRefused = errors.New("a step's end the runner does not take")
|
|
|
|
// ErrChanged is an Updater's answer to an Update whose revision is no longer the key's: another write came between.
|
|
var ErrChanged = errors.New("the pipeline was written since it was read")
|
|
|
|
// Updater is the state pipelines read and written by compare-and-set on the revision read (§27).
|
|
type Updater interface {
|
|
Get(key string) (seat.Pipeline, uint64, error)
|
|
Update(key string, p seat.Pipeline, revision uint64) (uint64, error)
|
|
// Keys are the keys matching a filter, `*` one key part.
|
|
Keys(filter string) ([]string, error)
|
|
}
|
|
|
|
// Hearer takes the steps' ends into their pipelines and ends a pipeline whose every step ended (design 47 G §24, §27).
|
|
type Hearer struct {
|
|
Store Updater
|
|
// Say says one of the runner's events of a pipeline, on its subject; Log appends to the step log.
|
|
Say func(subject string, body any) error
|
|
Log func(seat.LogEntry) error
|
|
// Keep writes one package's passing result into the state kept, under its key (§27): only from a step's end the
|
|
// runner took, so a kept result is its seat holder's alone.
|
|
Keep func(key string, k seat.KeptResult) error
|
|
// Output keeps a step's output under key in the objects step-outputs.
|
|
Output func(key string, data []byte) error
|
|
// Asker publishes an approval step's ask (§37); Name is the runner's own name, the asker a warrant names.
|
|
Asker Asker
|
|
Name string
|
|
// Where reads the router's record of one of the runner's asks, as operatorchannelclient.Client.Where does: its state and
|
|
// its warrant (design 47 G A-89, §700).
|
|
Where func(ask string) (string, *oc.Warrant, error)
|
|
Logf func(string, ...any)
|
|
Now func() time.Time
|
|
}
|
|
|
|
func (h *Hearer) now() time.Time {
|
|
if h.Now != nil {
|
|
return h.Now().UTC()
|
|
}
|
|
return time.Now().UTC()
|
|
}
|
|
|
|
// OnStepEnded takes the step-ended heard on subject, from seat's holders. It is refused, and said, when the subject's
|
|
// node is not the one the body names, when no pipeline the runner keeps names the ask, or when the ask's step names
|
|
// another seat, commit, fingerprint, module or step (ADR 0666 §21; the bus grants step-ended.<node> to that node's
|
|
// holder alone, so the subject's node is the account's). A step already ended takes nothing (the bus gives an event again).
|
|
func (h *Hearer) OnStepEnded(bs StepSeat, subject string, ev StepEnded) (seat.Pipeline, error) {
|
|
node := strings.TrimPrefix(subject, bs.Prefix)
|
|
if node == subject || node == "" || strings.Contains(node, ".") {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: %s is not a subject of %s's step-ended", ErrRefused, subject, bs.Seat)
|
|
}
|
|
if ev.Node != node {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s ended on %s's subject names node %q", ErrRefused, ev.ID, node, ev.Node)
|
|
}
|
|
for {
|
|
p, rev, err := h.Store.Get(ev.Pipeline)
|
|
if err != nil {
|
|
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
|
|
}
|
|
i := stepOfAsk(p, ev.ID)
|
|
switch {
|
|
case i < 0:
|
|
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
|
|
case p.Steps[i].Seat != bs.Seat:
|
|
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
|
|
case ev.Commit != p.Commit || ev.Fingerprint != p.Fingerprint || ev.Module != p.Module || ev.Step != p.Steps[i].Step:
|
|
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
|
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Commit, p.Fingerprint)
|
|
case p.Steps[i].Outcome != seat.StepResultOutcomeRunning:
|
|
return p, nil
|
|
case !endOutcome(ev.Outcome):
|
|
return p, fmt.Errorf("%w: ask %s ended %q, not an outcome of step-ended", ErrRefused, ev.ID, ev.Outcome)
|
|
}
|
|
now := h.now()
|
|
s := &p.Steps[i]
|
|
s.Outcome, s.Node, s.Why, s.Rerun, s.By, s.Ended = ev.Outcome, node, ev.Said, ev.Rerun, ev.By, &now
|
|
if ev.Tests != nil {
|
|
s.Tests = &seat.TestCounts{Unit: ev.Tests.Unit, Integration: ev.Tests.Integration}
|
|
}
|
|
if ev.Output != "" && h.Output != nil {
|
|
name := seat.StepOutputsByPipelineStepArtifactKey(keys.EscapeName(ev.Pipeline), s.Step, keys.EscapeName(s.Artifact))
|
|
if err := h.Output(name, []byte(ev.Output)); err != nil {
|
|
return p, fmt.Errorf("the output of ask %s cannot be kept: %w", ev.ID, err)
|
|
}
|
|
s.Output = name
|
|
}
|
|
from := now
|
|
if ev.Ended != nil {
|
|
from = ev.Ended.UTC()
|
|
}
|
|
asked, err := askApproval(ev.Pipeline, &p, from, h.Asker)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
ended := endPipeline(&p, now)
|
|
if _, err := h.Store.Update(ev.Pipeline, p, rev); errors.Is(err, ErrChanged) {
|
|
continue
|
|
} else if err != nil {
|
|
return p, fmt.Errorf("pipeline %s cannot be written: %w", ev.Pipeline, err)
|
|
}
|
|
if h.Keep != nil {
|
|
for _, k := range ev.Kept {
|
|
kept := seat.KeptResult{Digest: k.Digest, Level: k.Level, Package: k.Package, Pipeline: k.Pipeline}
|
|
if k.Toolchain != nil {
|
|
kept.Toolchain = &seat.ToolchainUsed{Language: k.Toolchain.Language, Recipe: k.Toolchain.Recipe, Version: k.Toolchain.Version}
|
|
}
|
|
if kept.Pipeline == "" {
|
|
kept.Pipeline = ev.Pipeline
|
|
}
|
|
if err := h.Keep(seat.KeptByLevelDigestKey(keys.EscapeName(k.Level), keys.EscapeName(k.Digest)), kept); err != nil {
|
|
h.logf("the passing result of %s (%s) at %s is not kept, so it runs again next time: %v", k.Package, k.Level, k.Digest, err)
|
|
}
|
|
}
|
|
}
|
|
h.logEnd(ev.Pipeline, *s, now)
|
|
if err := h.say(seat.EventStepEnded, p, s); err != nil {
|
|
return p, fmt.Errorf("pipeline %s's %s is written but not said: %w", ev.Pipeline, seat.EventStepEnded, err)
|
|
}
|
|
return p, h.said(p, asked, ended)
|
|
}
|
|
}
|
|
|
|
// said says what a written pipeline became: an approval asked is a wait held on the operator (ADR 0283), never a stall.
|
|
func (h *Hearer) said(p seat.Pipeline, asked, ended bool) error {
|
|
if asked {
|
|
if err := h.say(seat.EventWaitHeld, p, nil); err != nil {
|
|
return fmt.Errorf("pipeline %s waits on the operator but %s is not said: %w", p.Pipeline, seat.EventWaitHeld, err)
|
|
}
|
|
}
|
|
if ended {
|
|
if err := h.say(seat.EventPipelineEnded, p, nil); err != nil {
|
|
return fmt.Errorf("pipeline %s ended but %s is not said: %w", p.Pipeline, seat.EventPipelineEnded, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (h *Hearer) logEnd(key string, s seat.StepResult, at time.Time) {
|
|
if h.Log == nil {
|
|
return
|
|
}
|
|
if err := h.Log(seat.LogEntry{Pipeline: key, Step: s.Step, Artifact: s.Artifact, What: seat.LogEntryWhatEnded, Outcome: s.Outcome,
|
|
Output: s.Output, At: at}); err != nil {
|
|
h.logf("the step log misses the end of %s of %s: %v", s.Step, key, err)
|
|
}
|
|
}
|
|
|
|
func (h *Hearer) logf(format string, args ...any) {
|
|
if h.Logf != nil {
|
|
h.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// say says one event of p on the runner's subject for it; step, when not nil, is the result a step-ended carries.
|
|
func (h *Hearer) say(event string, p seat.Pipeline, step *seat.StepResult) error {
|
|
last := p.Commit
|
|
if p.Group != "" {
|
|
last = p.Group
|
|
}
|
|
subject := "mesh.seat." + seat.Seat + ".event." + event + "." + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Branch) + "." + keys.EscapeName(last)
|
|
at := h.now()
|
|
switch event {
|
|
case seat.EventStepEnded, seat.EventStepStarted:
|
|
return h.Say(subject, seat.StepEvent{V: 1, At: &at, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit,
|
|
Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Source: p.Source, Result: *step})
|
|
case seat.EventPipelineEnded:
|
|
return h.Say(subject, pipelineEnded(p))
|
|
default:
|
|
return h.Say(subject, seat.Wait{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
|
|
Group: p.Group, Revision: p.Revision, Source: p.Source, Step: seat.StepResultStepApproval, On: "operator", Since: at})
|
|
}
|
|
}
|
|
|
|
func pipelineEnded(p seat.Pipeline) seat.PipelineEnded {
|
|
return seat.PipelineEnded{V: 1, Pipeline: p.Pipeline, Branch: p.Branch, Module: p.Module, Commit: p.Commit, Fingerprint: p.Fingerprint,
|
|
Group: p.Group, Revision: p.Revision, Source: p.Source, Run: p.Run, Outcome: p.Outcome, Why: p.Why, Started: p.Started,
|
|
Ended: p.Ended, EndCount: p.EndCount, Steps: p.Steps}
|
|
}
|
|
|
|
func endOutcome(o string) bool {
|
|
switch o {
|
|
case nodebuildgo.StepEndedOutcomePassed, nodebuildgo.StepEndedOutcomeFailed, nodebuildgo.StepEndedOutcomeNothingToRun,
|
|
nodebuildgo.StepEndedOutcomeCouldNotRun:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func stepOfAsk(p seat.Pipeline, id string) int {
|
|
for i, s := range p.Steps {
|
|
if id != "" && s.Ask == id {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// endPipeline ends a pipeline whose every step ended: passed when every required step passed or was carried, else
|
|
// failed, naming each required step that held it (ADR 0666 §7, §9; design 47 G §24, §26). A failed optional step, and an optional step
|
|
// still waiting, hold nothing. An approval step still waiting when a required step did not pass is `not asked` (§26). It says whether the
|
|
// pipeline ended now, its end count one higher.
|
|
func endPipeline(p *seat.Pipeline, now time.Time) bool {
|
|
if p.Outcome != seat.PipelineOutcomeRunning {
|
|
return false
|
|
}
|
|
var held []string
|
|
notAsked := -1
|
|
for i, s := range p.Steps {
|
|
switch {
|
|
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
|
notAsked = i
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting && s.Status != seat.StepResultStatusRequired:
|
|
// An optional step still waiting (a code review nobody has given) holds nothing: only a required one holds the
|
|
// pipeline's end (A-104).
|
|
continue
|
|
case s.Outcome == seat.StepResultOutcomeWaiting, s.Outcome == seat.StepResultOutcomeRunning:
|
|
return false
|
|
}
|
|
if s.Status == seat.StepResultStatusRequired && s.Outcome != seat.StepResultOutcomePassed && s.Outcome != seat.StepResultOutcomeCarried {
|
|
held = append(held, s.Artifact+" "+s.Step+" "+s.Outcome)
|
|
}
|
|
}
|
|
if notAsked >= 0 {
|
|
if len(held) == 0 {
|
|
return false
|
|
}
|
|
p.Steps[notAsked].Outcome = seat.StepResultOutcomeNotAsked
|
|
}
|
|
p.Ended = &now
|
|
p.EndCount++
|
|
p.Outcome, p.Why = seat.PipelineOutcomePassed, ""
|
|
if len(held) > 0 {
|
|
p.Outcome, p.Why = seat.PipelineOutcomeFailed, "required steps not passed: "+strings.Join(held, ", ")
|
|
}
|
|
return true
|
|
}
|