Files
mesh-delivery/modules/mesh-delivery-pipeline/internal/pipeline/start.go
T
jschoubben 960c8e104a §416, §25, A-104: a step ask carries the kept results of its package and the toolchain of its language
The asking half reads, in one pass for all a pipeline's asks, the kept entries whose package is the artifact's package from
the catalog entry or lies beneath it, and the runner's own toolchain per language, kept in the new state toolchains as
heard from toolchain-agreed.

Builds: design 47 G §25, §416, A-104

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-11 23:43:35 +01:00

303 lines
12 KiB
Go

package pipeline
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"time"
runnerclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient"
builds "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
nodebuildgo "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54).
var ErrExists = errors.New("the pipeline's key is written already")
// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on
// its absence), ErrExists when it is not.
type Created interface {
Create(key string, value []byte) error
}
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
// what the catalog half of that branch's view says of the module.
type Begin struct {
Branch string
Source int64
Module string
Commit string
Fp string
Group string
Repository string
Revision int64
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
Entries map[string]catalog.CatalogEntry
}
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
// name is (§27).
func Key(branch string, source int64, module string) string {
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
}
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it
// wrote.
type Starter struct {
Pipelines Created
// Say says one of the seat's events on its subject.
Say func(subject string, body any) error
Now func() time.Time
Logf func(string, ...any)
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
// pipeline starts with no step and says so.
Registration func(module string) (catalog.Registration, bool)
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
// asker.
Steps StepAsker
Name string
// Kept reads the kept results of each named package, by the package name an artifact's catalog entry carries (§25, §416): the
// results whose package is that name or one beneath it. A step ask carries them, so a build seat runs only the packages
// with no result for their digest.
Kept func(packages []string) (map[string][]nodebuildgo.KeptResult, error)
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
// none was heard, which asks the step with none.
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
Arm Arm
}
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
// start writes nothing, §54).
func (s *Starter) Start(b Begin) (bool, error) {
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
key := Key(b.Branch, b.Source, b.Module)
p := seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp,
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
Steps: []seat.StepResult{}}
var asks []Asked
if s.Registration != nil {
if reg, ok := s.Registration(b.Module); ok {
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
} else {
s.logf("%s has no registration yet: its pipeline starts with no step", key)
}
}
raw, err := json.Marshal(p)
if err != nil {
return false, err
}
if err := s.Pipelines.Create(key, raw); errors.Is(err, ErrExists) {
s.logf("%s is started already: this start writes nothing", key)
return false, nil
} else if err != nil {
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
}
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision}
commitOrGroup := b.Commit
if b.Group != "" {
commitOrGroup = b.Group
}
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
s.logf("pipeline-started of %s is not said: %v", key, err)
}
if s.Steps != nil && len(asks) > 0 {
s.carry(key, asks)
if err := askAll(s.Steps, s.Arm, key, asks); err != nil {
return true, err
}
}
return true, nil
}
// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still
// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped
// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was
// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts
// and seats as Place reads it. It says how many steps it asked.
func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
if s.Steps == nil || s.Registration == nil {
return 0, nil
}
pkeys, err := store.Keys("*.*.*")
if err != nil {
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
}
now := time.Now().UTC()
if s.Now != nil {
now = s.Now().UTC()
}
asked := 0
var first error
for _, key := range pkeys {
p, _, err := store.Get(key)
if err != nil {
if first == nil {
first = fmt.Errorf("the pipeline %s is not read: %w", key, err)
}
continue
}
if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning {
continue
}
reg, ok := s.Registration(p.Module)
if !ok {
continue
}
running := map[string]bool{}
for _, st := range p.Steps {
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
running[st.Ask] = true
}
}
_, placed := Place(p, reg, half.Entries, now, s.Name)
var due []Asked
for _, a := range placed {
if running[a.ID] {
due = append(due, a)
}
}
if len(due) == 0 {
continue
}
s.carry(key, due)
if err := askAll(s.Steps, s.Arm, key, due); err != nil && first == nil {
first = err
}
asked += len(due)
}
return asked, first
}
// carry puts the kept results of each ask's package and the toolchain of its language on its step ask (§25, §416, A-104). A
// read that fails asks the steps with none, so every package runs: slower, never wrong; it is said.
func (s *Starter) carry(key string, asks []Asked) {
var pkgs []string
seen := map[string]bool{}
for _, a := range asks {
if a.Package != "" && !seen[a.Package] {
seen[a.Package] = true
pkgs = append(pkgs, a.Package)
}
}
var kept map[string][]nodebuildgo.KeptResult
if s.Kept != nil && len(pkgs) > 0 {
var err error
if kept, err = s.Kept(pkgs); err != nil {
s.logf("the kept results for the steps of %s are not read, so each package runs: %v", key, err)
}
}
chains := map[string]*nodebuildgo.ToolchainUsed{}
for _, a := range asks {
ask, ok := a.Ask.(nodebuildgo.StepAsk)
if !ok || ask.Step == nil {
continue
}
if a.Package != "" {
ask.Step.Kept = kept[a.Package]
}
if s.Toolchain == nil || ask.Language == "" {
continue
}
t, done := chains[ask.Language]
if !done {
var err error
if t, err = s.Toolchain(ask.Language); err != nil {
s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err)
}
chains[ask.Language] = t
}
ask.Step.Toolchain = t
}
}
func (s *Starter) logf(format string, args ...any) {
if s.Logf != nil {
s.Logf(format, args...)
}
}
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
// a package A imports (§53).
const outdatedKind = 1
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
// dependent the half does not hold is said and started with what the event names.
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
}
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
if err != nil {
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
}
half, _ := view(ev.Branch)
started := 0
var first error
for _, o := range a.Behind {
if o.Kind != outdatedKind || o.Module == "" {
continue
}
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision, Entries: half.Entries}
if e, ok := half.Entries[o.Module]; ok {
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
} else {
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
}
ok, err := s.Start(b)
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
return started, first
}
// TakenInSubject is where any module's take-in.taken is said, `mesh.mod.<module>.event.take-in.taken`: the runner names no
// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54).
const TakenInSubject = "mesh.mod.*.event.take-in.taken"
// TakenIn starts, for a take-in.taken, one pipeline per entry of the source revision, keyed by its watched branch, that
// source revision and the module (design 47 G §44, §54, §476; ADR 0666 §13). A hand drop takes nothing in and starts
// nothing; a second start of an entry, a package-published having started it first, writes nothing (§54). view is the
// catalog half of the event's branch, read only for the revision its pipelines are made from.
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
if ev.Dropped {
s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By)
return 0, nil
}
if ev.Branch == "" || ev.Source <= 0 {
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source)
}
half, _ := view(ev.Branch)
started := 0
var first error
for _, e := range ev.Entries {
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
Group: ev.Group, Repository: e.Repository, Revision: half.Revision, Entries: half.Entries})
if err != nil && first == nil {
first = err
}
if ok {
started++
}
}
return started, first
}