A batched take-in names only its newest revision's movers; the other modules it moved never got a pipeline. Starter.TakenIn and the runner's first-half pickup now start one for each module of the branch's head with no pipeline at its fingerprint. Builds: design 47 G §44, §54, §88, §476
551 lines
23 KiB
Go
551 lines
23 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"time"
|
|
|
|
runnerclient "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
|
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
|
"git.novox.be/novox/mesh-sdk/go/keys"
|
|
)
|
|
|
|
// ErrExists is a Create of a pipeline key that is written already: the second start, which writes nothing (§54).
|
|
var ErrExists = errors.New("the pipeline's key is written already")
|
|
|
|
// Created is the runner's state pipelines as a start writes it: a key written only when it is absent (compare-and-set on
|
|
// its absence), ErrExists when it is not.
|
|
type Created interface {
|
|
Create(key string, value []byte) error
|
|
}
|
|
|
|
// Begin is what a start names: the pipeline's key (watched branch, source revision, module; design 47 G §27, §54) and
|
|
// what the catalog half of that branch's view says of the module.
|
|
type Begin struct {
|
|
Branch string
|
|
Source int64
|
|
Module string
|
|
Commit string
|
|
Fp string
|
|
Group string
|
|
Repository string
|
|
Revision int64
|
|
// Entries are the catalog half's entries of the pipeline's branch: the steps' artifacts and build seats are read from them.
|
|
Entries map[string]catalog.CatalogEntry
|
|
// Published is the package's publish that started the pipeline, nil for any other start (A-202).
|
|
Published *seat.Published
|
|
// unheld says the branch's catalog half is not held yet: the pipeline starts with no steps, placed once it is (A-336).
|
|
unheld bool
|
|
}
|
|
|
|
// Key is a pipeline's key in the state pipelines, `<branch>.<source revision>.<module>`, the branch escaped as a key's
|
|
// name is (§27).
|
|
func Key(branch string, source int64, module string) string {
|
|
return seat.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), module)
|
|
}
|
|
|
|
// Starter starts pipelines (design 47 G §54, §103): it writes a pipeline's key by compare-and-set on its absence, so one
|
|
// source revision and module has one pipeline whichever event starts it first, and says pipeline-started for the one it
|
|
// wrote.
|
|
type Starter struct {
|
|
Pipelines Created
|
|
// Say says one of the seat's events on its subject.
|
|
Say func(subject string, body any) error
|
|
Now func() time.Time
|
|
Logf func(string, ...any)
|
|
// Registration is a module's registration, whose policy names the steps of its pipeline (§21, §23); with none, a
|
|
// pipeline starts with no step and says so.
|
|
Registration func(module string) (catalog.Registration, bool)
|
|
// Steps asks each build seat for its step, as the pipeline's asking half (§22, §24); Name is the runner's own name as an
|
|
// asker.
|
|
Steps StepAsker
|
|
Name string
|
|
// Kept reads the kept results of each named package, by the package name an artifact's catalog entry carries (§25, §416): the
|
|
// results whose package is that name or one beneath it. A step ask carries them, so a build seat runs only the packages
|
|
// with no result for their digest.
|
|
Kept func(packages []string) (map[string][]nodebuildgo.KeptResult, error)
|
|
// Toolchain is the toolchain of a language as the runner heard it agreed (§416, A-104): its language and version; nil when
|
|
// none was heard, which asks the step with none.
|
|
Toolchain func(language string) (*nodebuildgo.ToolchainUsed, error)
|
|
// Arm arms the bound of an ask put on a build seat's work queue (§535, §647); nil arms none.
|
|
Arm Arm
|
|
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
|
Fills *Fills
|
|
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
|
Recovered func(commit string) (bool, error)
|
|
// Store is the state pipelines read and written by compare-and-set, where a run's pipelines are placed (PlaceRun).
|
|
Store Updater
|
|
// Supersede ends the older running pipelines the pipeline just written supersedes (§31; Hearer.Supersede); nil ends none.
|
|
Supersede func(newer seat.Pipeline)
|
|
}
|
|
|
|
func newPipeline(b Begin, key string, now time.Time) seat.Pipeline {
|
|
return seat.Pipeline{V: 1, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module, Commit: b.Commit, Fingerprint: b.Fp,
|
|
Group: b.Group, Repository: b.Repository, Revision: b.Revision, Outcome: seat.PipelineOutcomeRunning, Started: &now,
|
|
Steps: []seat.StepResult{}, Published: b.Published}
|
|
}
|
|
|
|
func (s *Starter) create(key string, p seat.Pipeline) error {
|
|
raw, err := json.Marshal(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return s.Pipelines.Create(key, raw)
|
|
}
|
|
|
|
// Start writes b's pipeline, running, and says pipeline-started; false, nil when its key was written already (the second
|
|
// start writes nothing, §54).
|
|
func (s *Starter) Start(b Begin) (bool, error) {
|
|
if b.Branch == "" || b.Module == "" || b.Source <= 0 {
|
|
return false, fmt.Errorf("a pipeline names its watched branch, source revision and module: %+v", b)
|
|
}
|
|
now := time.Now().UTC()
|
|
if s.Now != nil {
|
|
now = s.Now().UTC()
|
|
}
|
|
key := Key(b.Branch, b.Source, b.Module)
|
|
if err := s.Fills.refuseStart(key); err != nil {
|
|
return false, err
|
|
}
|
|
p := newPipeline(b, key, now)
|
|
var asks []Asked
|
|
if b.unheld {
|
|
s.logf("%s starts with no step: the catalog half of %s is not held yet, and its steps are placed once it is (A-336)", key, b.Branch)
|
|
} else if s.Registration != nil {
|
|
if reg, ok := s.Registration(b.Module); ok {
|
|
p.Steps, asks = Place(p, reg, b.Entries, now, s.Name)
|
|
} else {
|
|
s.logf("%s has no registration yet: its pipeline starts with no step and waits on the registration of %s, which places them (A-327)", key, b.Module)
|
|
}
|
|
}
|
|
if err := s.create(key, p); errors.Is(err, ErrExists) {
|
|
s.logf("%s is started already: this start writes nothing", key)
|
|
return false, nil
|
|
} else if err != nil {
|
|
return false, fmt.Errorf("the pipeline %s is not written: %w", key, err)
|
|
}
|
|
s.pin(p) // the catalog graph revision it runs on is kept while it runs (A-449)
|
|
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: b.Branch, Source: b.Source, Module: b.Module,
|
|
Commit: b.Commit, Fingerprint: b.Fp, Group: b.Group, Revision: b.Revision, Published: b.Published}
|
|
commitOrGroup := b.Commit
|
|
if b.Group != "" {
|
|
commitOrGroup = b.Group
|
|
}
|
|
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, b.Module, b.Branch, commitOrGroup), ev); err != nil {
|
|
s.logf("pipeline-started of %s is not said: %v", key, err)
|
|
}
|
|
if s.Supersede != nil {
|
|
s.Supersede(p)
|
|
}
|
|
if s.Steps != nil && len(asks) > 0 {
|
|
s.carry(key, asks)
|
|
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, b.Commit)); err != nil {
|
|
return true, err
|
|
}
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Resume asks again, of the build seats and the assigner, each step of branch's running pipelines that is placed and still
|
|
// running (design 47 G §88's restart pickup): a pipeline written whose asks never reached their queues (the runner stopped
|
|
// between its write and its asks, or a publish failed) is asked at the next start. Each ask carries the id its step was
|
|
// placed under, so one the seat holds already is kept once. half is the branch's catalog half, read for the steps' artifacts
|
|
// and seats as Place reads it. It says how many steps it asked.
|
|
func (s *Starter) Resume(store Updater, branch string, half Half) (int, error) {
|
|
if s.Steps == nil || s.Registration == nil {
|
|
return 0, nil
|
|
}
|
|
pkeys, err := store.Keys("*.*.*")
|
|
if err != nil {
|
|
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
|
}
|
|
now := time.Now().UTC()
|
|
if s.Now != nil {
|
|
now = s.Now().UTC()
|
|
}
|
|
asked := 0
|
|
var first error
|
|
for _, key := range pkeys {
|
|
p, _, err := store.Get(key)
|
|
if err != nil {
|
|
if first == nil {
|
|
first = fmt.Errorf("the pipeline %s is not read: %w", key, err)
|
|
}
|
|
continue
|
|
}
|
|
// A run on request's steps are placed from its candidate, not from the branch's half (PlaceRun).
|
|
if p.Branch != branch || p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" {
|
|
continue
|
|
}
|
|
reg, ok := s.Registration(p.Module)
|
|
if !ok {
|
|
continue
|
|
}
|
|
running := map[string]bool{}
|
|
for _, st := range p.Steps {
|
|
if st.Outcome == seat.StepResultOutcomeRunning && st.Ask != "" {
|
|
running[st.Ask] = true
|
|
}
|
|
}
|
|
_, placed := Place(p, reg, half.Entries, now, s.Name)
|
|
var due []Asked
|
|
for _, a := range placed {
|
|
if running[a.ID] {
|
|
due = append(due, a)
|
|
}
|
|
}
|
|
if len(due) == 0 {
|
|
continue
|
|
}
|
|
s.carry(key, due)
|
|
if err := askAll(s.Steps, s.Arm, key, due, s.recovered(key, p.Commit)); err != nil && first == nil {
|
|
first = err
|
|
}
|
|
asked += len(due)
|
|
}
|
|
return asked, first
|
|
}
|
|
|
|
// Registered places, on a registration-changed of reg's module, the steps of every running pipeline of that module that has
|
|
// none, from the registration (design 47 G §21, §23, A-327): a pipeline started before its module had a registration. Each is
|
|
// written by compare-and-set, so it is placed once, its steps asked, and said; one held for the runner's new version is left to
|
|
// its release, a run on request to its candidate, and one whose branch has no catalog half yet keeps waiting, said, until that
|
|
// half is held (Waiting). It says how many it placed.
|
|
func (s *Starter) Registered(store Updater, reg catalog.Registration, view func(branch string) (Half, bool)) (int, error) {
|
|
if reg.Module == "" {
|
|
return 0, nil
|
|
}
|
|
return s.placeWaiting(store, "*.*."+reg.Module, func(p seat.Pipeline) (catalog.Registration, bool) {
|
|
return reg, p.Module == reg.Module
|
|
}, view, "the registration of "+reg.Module+", which arrived after the pipeline started")
|
|
}
|
|
|
|
// Waiting places, from the registrations held now, the steps of every running pipeline of branch that has none (A-327): a
|
|
// registration heard before the runner could place from it, or before the branch's catalog half was held, is not lost, since
|
|
// the half becoming held and the registrations' first view being held each place what waited on them. An empty branch is
|
|
// every branch. It says how many it placed.
|
|
func (s *Starter) Waiting(store Updater, branch string, view func(branch string) (Half, bool)) (int, error) {
|
|
if s.Registration == nil {
|
|
return 0, nil
|
|
}
|
|
return s.placeWaiting(store, "*.*.*", func(p seat.Pipeline) (catalog.Registration, bool) {
|
|
if branch != "" && p.Branch != branch {
|
|
return catalog.Registration{}, false
|
|
}
|
|
return s.Registration(p.Module)
|
|
}, view, "the registration held when its branch's half or the registrations became held")
|
|
}
|
|
|
|
// placeWaiting places the steps of each running pipeline under pattern that has none and is not a run on request,
|
|
// and whose registration reg names, from that registration and its branch's catalog half (Registered, Waiting).
|
|
func (s *Starter) placeWaiting(store Updater, pattern string, reg func(seat.Pipeline) (catalog.Registration, bool), view func(branch string) (Half, bool), from string) (int, error) {
|
|
pkeys, err := store.Keys(pattern)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("the pipelines %s are not listed: %w", pattern, err)
|
|
}
|
|
now := time.Now().UTC()
|
|
if s.Now != nil {
|
|
now = s.Now().UTC()
|
|
}
|
|
placed := 0
|
|
var first error
|
|
note := func(err error) {
|
|
if first == nil {
|
|
first = err
|
|
}
|
|
}
|
|
for _, key := range pkeys {
|
|
p, rev, err := store.Get(key)
|
|
if err != nil {
|
|
note(fmt.Errorf("the pipeline %s is not read: %w", key, err))
|
|
continue
|
|
}
|
|
if p.Outcome != seat.PipelineOutcomeRunning || p.Run != "" || len(p.Steps) > 0 {
|
|
continue
|
|
}
|
|
r, ok := reg(p)
|
|
if !ok {
|
|
continue
|
|
}
|
|
half, ok := view(p.Branch)
|
|
if !ok {
|
|
s.logf("the catalog half of %s is not held yet: %s waits on it for its steps, placed once it is held", p.Branch, key)
|
|
continue
|
|
}
|
|
if p.Revision > 0 && half.Revision < p.Revision {
|
|
s.logf("the catalog half of %s holds revision %d: %s waits for revision %d, which its take-in made, to place its steps (§200)", p.Branch, half.Revision, key, p.Revision)
|
|
continue
|
|
}
|
|
var asks []Asked
|
|
p.Steps, asks = Place(p, r, half.Entries, now, s.Name)
|
|
if _, err := store.Update(key, p, rev); err != nil {
|
|
s.logf("%s is written meanwhile, so nothing is placed in it now: %v", key, err)
|
|
continue
|
|
}
|
|
placed++
|
|
s.logf("%s: %d steps placed from %s (A-327)", key, len(p.Steps), from)
|
|
// A pipeline whose steps were never asked is started by this placing, one a runner before this one set aside included
|
|
// (A-446a); the pipeline-started of its start, said already, is said again harmlessly.
|
|
if s.Say != nil {
|
|
ev := seat.PipelineStarted{V: 1, At: &now, Pipeline: key, Branch: p.Branch, Source: p.Source, Module: p.Module,
|
|
Commit: p.Commit, Fingerprint: p.Fingerprint, Group: p.Group, Revision: p.Revision, Published: p.Published}
|
|
if err := s.Say(runnerclient.Subject(seat.EventPipelineStarted, p.Module, p.Branch, commitOrGroup(p)), ev); err != nil {
|
|
s.logf("pipeline-started of %s is not said: %v", key, err)
|
|
}
|
|
}
|
|
if s.Steps != nil && len(asks) > 0 {
|
|
s.carry(key, asks)
|
|
if err := askAll(s.Steps, s.Arm, key, asks, s.recovered(key, p.Commit)); err != nil {
|
|
note(err)
|
|
}
|
|
}
|
|
}
|
|
return placed, first
|
|
}
|
|
|
|
// recovered says whether the commit of pipeline key is named a recovery; a read that fails asks on the stream check, said, since
|
|
// a step is asked either way (a recovery's only goes ahead of the first round).
|
|
func (s *Starter) recovered(key, commit string) bool {
|
|
if s.Recovered == nil || commit == "" {
|
|
return false
|
|
}
|
|
named, err := s.Recovered(commit)
|
|
if err != nil {
|
|
s.logf("whether %s of %s is a recovery is not read, so its steps are asked on check: %v", commit, key, err)
|
|
}
|
|
return named
|
|
}
|
|
|
|
// carry puts the kept results of each ask's package and the toolchain of its language on its step ask (§25, §416, A-104). A
|
|
// read that fails asks the steps with none, so every package runs: slower, never wrong; it is said.
|
|
func (s *Starter) carry(key string, asks []Asked) {
|
|
var pkgs []string
|
|
seen := map[string]bool{}
|
|
for _, a := range asks {
|
|
if a.Package != "" && !seen[a.Package] {
|
|
seen[a.Package] = true
|
|
pkgs = append(pkgs, a.Package)
|
|
}
|
|
}
|
|
var kept map[string][]nodebuildgo.KeptResult
|
|
if s.Kept != nil && len(pkgs) > 0 {
|
|
var err error
|
|
if kept, err = s.Kept(pkgs); err != nil {
|
|
s.logf("the kept results for the steps of %s are not read, so each package runs: %v", key, err)
|
|
}
|
|
}
|
|
chains := map[string]*nodebuildgo.ToolchainUsed{}
|
|
for _, a := range asks {
|
|
ask, ok := a.Ask.(nodebuildgo.StepAsk)
|
|
if !ok || ask.Step == nil {
|
|
continue
|
|
}
|
|
if a.Package != "" {
|
|
ask.Step.Kept = kept[a.Package]
|
|
}
|
|
if s.Toolchain == nil || ask.Language == "" {
|
|
continue
|
|
}
|
|
t, done := chains[ask.Language]
|
|
if !done {
|
|
var err error
|
|
if t, err = s.Toolchain(ask.Language); err != nil {
|
|
s.logf("the toolchain of %s for the steps of %s is not read: %v", ask.Language, key, err)
|
|
}
|
|
chains[ask.Language] = t
|
|
}
|
|
ask.Step.Toolchain = t
|
|
}
|
|
}
|
|
|
|
func (s *Starter) logf(format string, args ...any) {
|
|
if s.Logf != nil {
|
|
s.Logf(format, args...)
|
|
}
|
|
}
|
|
|
|
// outdatedKind is the edge `outdated` names a dependent behind by for a rebuild on a publish: B published before A builds,
|
|
// a package A imports (§53).
|
|
const outdatedKind = 1
|
|
|
|
// Published starts, for a package-published, a pipeline for each dependent the catalog's `outdated` names behind for that
|
|
// package on that branch by kind 1, keyed by the publication's watched branch and source revision and the dependent
|
|
// (design 47 G §53, §334). outdated is the kept call of the verb; view the catalog half of the publication's branch. A
|
|
// dependent the half does not hold is said and started with what the event names.
|
|
func (s *Starter) Published(ev builds.PackagePublished, outdated func(catalog.OutdatedInput) (catalog.OutdatedAnswer, error), view func(branch string) (Half, bool)) (int, error) {
|
|
if ev.Artifact == "" || ev.Branch == "" || ev.Source <= 0 {
|
|
return 0, fmt.Errorf("a package-published names its artifact, watched branch and source revision: %+v", ev)
|
|
}
|
|
a, err := outdated(catalog.OutdatedInput{Branch: ev.Branch, Dependency: ev.Artifact})
|
|
if err != nil {
|
|
return 0, fmt.Errorf("outdated of %s on %s: %w", ev.Artifact, ev.Branch, err)
|
|
}
|
|
// A publish heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries
|
|
// (§200, A-336); Waiting places them once the half is held.
|
|
half, held := view(ev.Branch)
|
|
started := 0
|
|
var first error
|
|
for _, o := range a.Behind {
|
|
if o.Kind != outdatedKind || o.Module == "" {
|
|
continue
|
|
}
|
|
b := Begin{Branch: ev.Branch, Source: ev.Source, Module: o.Module, Revision: half.Revision, Entries: half.Entries,
|
|
Published: &seat.Published{Artifact: ev.Artifact, Version: ev.Version}, unheld: !held}
|
|
if e, ok := half.Entries[o.Module]; ok {
|
|
b.Commit, b.Fp, b.Repository = e.Commit, e.Fp, e.Repository
|
|
} else if held {
|
|
s.logf("the catalog half of %s holds no entry of %s, behind %s: its pipeline starts without its commit", ev.Branch, o.Module, ev.Artifact)
|
|
}
|
|
ok, err := s.Start(b)
|
|
if err != nil && first == nil {
|
|
first = err
|
|
}
|
|
if ok {
|
|
started++
|
|
}
|
|
}
|
|
return started, first
|
|
}
|
|
|
|
// TakenInSubject is where any module's take-in.taken is said, `mesh.mod.<module>.event.take-in.taken`: the runner names no
|
|
// module (§7), so the bus's grant from the manifest's `consumes` says which one it hears (§54).
|
|
const TakenInSubject = "mesh.mod.*.event.take-in.taken"
|
|
|
|
// TakenIn starts, for a take-in.taken, one pipeline per entry of the source revision, keyed by its watched branch, that
|
|
// source revision and the module (design 47 G §44, §54, §476; ADR 0666 §13). A hand drop takes nothing in and starts
|
|
// nothing; a second start of an entry, a package-published having started it first, writes nothing (§54). view is the
|
|
// catalog half of the event's branch, read only for the revision its pipelines are made from.
|
|
func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half, bool)) (int, error) {
|
|
if ev.Dropped {
|
|
s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By)
|
|
return 0, nil
|
|
}
|
|
if len(ev.Entries) == 0 {
|
|
// A taken event that took nothing in (a group's end, one said before G and read again) starts nothing (§493).
|
|
s.logf("the take-in %s names no entries: it starts no pipeline", ev.Key)
|
|
return 0, nil
|
|
}
|
|
if ev.Branch == "" || ev.Source <= 0 {
|
|
return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source)
|
|
}
|
|
// A take-in heard before its branch's half is held starts its pipelines with no steps, never placed from empty entries
|
|
// (§200, A-336); Waiting places them once the half is held.
|
|
half, held := view(ev.Branch)
|
|
// The pipelines start at the revision the take-in made, never at the one the runner's own view holds (A-448): while the
|
|
// view holds an older one they start with no step, which Waiting places once the view reaches it (§200). An event with
|
|
// no revision (said before A-448) starts at the view's, as it did, and says so.
|
|
revision := ev.Revision
|
|
if revision <= 0 {
|
|
revision = half.Revision
|
|
s.logf("the take-in %s of %s names no catalog graph revision: its pipelines start at the revision the runner's view holds, %d", ev.Key, ev.Branch, revision)
|
|
} else if held && half.Revision < revision {
|
|
s.logf("the take-in %s of %s made revision %d and the runner's view holds %d: its pipelines start with no step, placed once the view reaches it (§200)", ev.Key, ev.Branch, revision, half.Revision)
|
|
held = false
|
|
}
|
|
started := 0
|
|
var first error
|
|
for _, e := range ev.Entries {
|
|
ok, err := s.Start(Begin{Branch: ev.Branch, Source: ev.Source, Module: e.Module, Commit: e.Commit, Fp: e.Fingerprint,
|
|
Group: ev.Group, Repository: e.Repository, Revision: revision, Entries: half.Entries, unheld: !held})
|
|
if err != nil && first == nil {
|
|
first = err
|
|
}
|
|
if ok {
|
|
started++
|
|
}
|
|
}
|
|
// A batch of merges is taken in as one event naming the newest revision's movers only; every other module the batch moved
|
|
// has its fingerprint in the half and no pipeline yet, and starts one here at the take-in's source revision.
|
|
if held {
|
|
n, err := s.StartUnstarted(ev.Branch, ev.Source, half)
|
|
started += n
|
|
if err != nil && first == nil {
|
|
first = err
|
|
}
|
|
}
|
|
// The view was read before the starts wrote: when pipelines were held back for it and it has reached the revision since,
|
|
// they are placed now, not left for an advance that already came (A-448, §200).
|
|
if !held && started > 0 && revision > 0 && s.Store != nil {
|
|
if again, ok := view(ev.Branch); ok && again.Revision >= revision {
|
|
if _, err := s.Waiting(s.Store, ev.Branch, view); err != nil && first == nil {
|
|
first = err
|
|
}
|
|
}
|
|
}
|
|
return started, first
|
|
}
|
|
|
|
func commitOrGroup(p seat.Pipeline) string {
|
|
if p.Group != "" {
|
|
return p.Group
|
|
}
|
|
return p.Commit
|
|
}
|
|
|
|
// Unstarted names the modules of half whose fingerprint has no pipeline on branch: no pipeline of any outcome, a published
|
|
// one included, carries that module at that fingerprint. A run on request is no pipeline of the branch's head. Sorted.
|
|
func (s *Starter) Unstarted(branch string, half Half) ([]string, error) {
|
|
if s.Store == nil {
|
|
return nil, errors.New("the runner holds no state pipelines to read")
|
|
}
|
|
pkeys, err := s.Store.Keys(keys.EscapeName(branch) + ".*.*")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the pipelines of %s are not listed: %w", branch, err)
|
|
}
|
|
have := map[string]bool{}
|
|
for _, k := range pkeys {
|
|
p, _, err := s.Store.Get(k)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the pipeline %s is not read: %w", k, err)
|
|
}
|
|
if p.Run == "" {
|
|
have[p.Module+"\x00"+p.Fingerprint] = true
|
|
}
|
|
}
|
|
var out []string
|
|
for module, e := range half.Entries {
|
|
if e.Fp != "" && !have[module+"\x00"+e.Fp] {
|
|
out = append(out, module)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out, nil
|
|
}
|
|
|
|
// StartUnstarted starts, at source, the pipeline of each module Unstarted names, from the entry half holds: the class fix of a
|
|
// batched take-in, which names only its newest revision's movers. It says how many it started.
|
|
func (s *Starter) StartUnstarted(branch string, source int64, half Half) (int, error) {
|
|
if s.Store == nil {
|
|
return 0, nil
|
|
}
|
|
if source <= 0 {
|
|
s.logf("the catalog half of %s names no source revision: the modules with no pipeline start none now", branch)
|
|
return 0, nil
|
|
}
|
|
modules, err := s.Unstarted(branch, half)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
started := 0
|
|
var first error
|
|
for _, m := range modules {
|
|
e := half.Entries[m]
|
|
ok, err := s.Start(Begin{Branch: branch, Source: source, Module: m, Commit: e.Commit, Fp: e.Fp, Repository: e.Repository,
|
|
Revision: half.Revision, Entries: half.Entries})
|
|
if err != nil && first == nil {
|
|
first = err
|
|
}
|
|
if ok {
|
|
started++
|
|
s.logf("%s has no pipeline at its fingerprint %s: one is started at source revision %d", m, e.Fp, source)
|
|
}
|
|
}
|
|
return started, first
|
|
}
|